2024-12-10 02:22:51,633 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-10 02:22:51,644 main DEBUG Took 0.009898 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-10 02:22:51,645 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-10 02:22:51,645 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-10 02:22:51,646 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-10 02:22:51,647 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,653 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-10 02:22:51,665 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,669 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,671 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,672 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,673 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,674 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,675 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,677 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,678 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,678 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,679 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,680 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 02:22:51,681 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,681 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-10 02:22:51,683 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 02:22:51,685 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-10 02:22:51,687 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-10 02:22:51,688 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-10 02:22:51,689 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-10 02:22:51,690 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-10 02:22:51,701 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-10 02:22:51,703 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-10 02:22:51,705 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-10 02:22:51,705 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-10 02:22:51,705 main DEBUG createAppenders(={Console})
2024-12-10 02:22:51,706 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-12-10 02:22:51,706 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-12-10 02:22:51,706 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-12-10 02:22:51,707 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-10 02:22:51,707 main DEBUG OutputStream closed
2024-12-10 02:22:51,707 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-10 02:22:51,708 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-10 02:22:51,708 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-12-10 02:22:51,785 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-10 02:22:51,788 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-10 02:22:51,789 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-10 02:22:51,790 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-10 02:22:51,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-10 02:22:51,791 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-10 02:22:51,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-10 02:22:51,792 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-10 02:22:51,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-10 02:22:51,793 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-10 02:22:51,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-10 02:22:51,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-10 02:22:51,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-10 02:22:51,794 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-10 02:22:51,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-10 02:22:51,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-10 02:22:51,795 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-10 02:22:51,796 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-10 02:22:51,798 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10 02:22:51,798 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-12-10 02:22:51,799 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-10 02:22:51,799 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-12-10T02:22:52,058 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b
2024-12-10 02:22:52,061 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-10 02:22:52,061 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10T02:22:52,071 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-12-10T02:22:52,102 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=160, ProcessCount=11, AvailableMemoryMB=4939
2024-12-10T02:22:52,105 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:22:52,121 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf, deleteOnExit=true
2024-12-10T02:22:52,122 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:22:52,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/test.cache.data in system properties and HBase conf
2024-12-10T02:22:52,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:22:52,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:22:52,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:22:52,125 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:22:52,125 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:22:52,222 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-10T02:22:52,307 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:22:52,310 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:22:52,311 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:22:52,311 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:22:52,312 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:22:52,312 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:22:52,313 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:22:52,313 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:22:52,313 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:22:52,314 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:22:52,314 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:22:52,315 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:22:52,315 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:22:52,315 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:22:52,316 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:22:52,794 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:22:53,145 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-12-10T02:22:53,242 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2299ms to org.eclipse.jetty.util.log.Slf4jLog
2024-12-10T02:22:53,327 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:22:53,389 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:22:53,408 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:22:53,409 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:22:53,410 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:22:53,423 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:22:53,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:22:53,426 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:22:53,629 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/java.io.tmpdir/jetty-localhost-37761-hadoop-hdfs-3_4_1-tests_jar-_-any-12427575772666012645/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:22:53,637 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:37761}
2024-12-10T02:22:53,637 INFO  [Time-limited test {}] server.Server(415): Started @2695ms
2024-12-10T02:22:53,672 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:22:54,015 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:22:54,022 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:22:54,023 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:22:54,023 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:22:54,023 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:22:54,024 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:22:54,024 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:22:54,144 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/java.io.tmpdir/jetty-localhost-38711-hadoop-hdfs-3_4_1-tests_jar-_-any-3348834237886420767/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:22:54,145 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:38711}
2024-12-10T02:22:54,145 INFO  [Time-limited test {}] server.Server(415): Started @3204ms
2024-12-10T02:22:54,202 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:22:54,322 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:22:54,329 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:22:54,333 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:22:54,333 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:22:54,333 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:22:54,335 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:22:54,335 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:22:54,498 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/java.io.tmpdir/jetty-localhost-33443-hadoop-hdfs-3_4_1-tests_jar-_-any-10269965887004570299/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:22:54,498 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:33443}
2024-12-10T02:22:54,499 INFO  [Time-limited test {}] server.Server(415): Started @3557ms
2024-12-10T02:22:54,501 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:22:54,665 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data2/current/BP-754960761-172.17.0.2-1733797372889/current, will proceed with Du for space computation calculation, 
2024-12-10T02:22:54,665 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data1/current/BP-754960761-172.17.0.2-1733797372889/current, will proceed with Du for space computation calculation, 
2024-12-10T02:22:54,665 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data4/current/BP-754960761-172.17.0.2-1733797372889/current, will proceed with Du for space computation calculation, 
2024-12-10T02:22:54,665 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data3/current/BP-754960761-172.17.0.2-1733797372889/current, will proceed with Du for space computation calculation, 
2024-12-10T02:22:54,721 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:22:54,722 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:22:54,795 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20efab466175d290 with lease ID 0x6eb1f02a33da5379: Processing first storage report for DS-66884d04-c986-46c8-adb8-6c58ee6d7e27 from datanode DatanodeRegistration(127.0.0.1:45883, datanodeUuid=850558d8-1678-4950-8047-bc393b287634, infoPort=42487, infoSecurePort=0, ipcPort=41361, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889)
2024-12-10T02:22:54,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20efab466175d290 with lease ID 0x6eb1f02a33da5379: from storage DS-66884d04-c986-46c8-adb8-6c58ee6d7e27 node DatanodeRegistration(127.0.0.1:45883, datanodeUuid=850558d8-1678-4950-8047-bc393b287634, infoPort=42487, infoSecurePort=0, ipcPort=41361, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-12-10T02:22:54,797 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfbede119ba09f8ed with lease ID 0x6eb1f02a33da537a: Processing first storage report for DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3 from datanode DatanodeRegistration(127.0.0.1:43857, datanodeUuid=39d3d08c-b056-404d-98b0-85dd3ad62ccf, infoPort=46657, infoSecurePort=0, ipcPort=33459, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889)
2024-12-10T02:22:54,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfbede119ba09f8ed with lease ID 0x6eb1f02a33da537a: from storage DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3 node DatanodeRegistration(127.0.0.1:43857, datanodeUuid=39d3d08c-b056-404d-98b0-85dd3ad62ccf, infoPort=46657, infoSecurePort=0, ipcPort=33459, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T02:22:54,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20efab466175d290 with lease ID 0x6eb1f02a33da5379: Processing first storage report for DS-b00b0852-8eb8-4c51-b3c3-96c4bf5490a9 from datanode DatanodeRegistration(127.0.0.1:45883, datanodeUuid=850558d8-1678-4950-8047-bc393b287634, infoPort=42487, infoSecurePort=0, ipcPort=41361, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889)
2024-12-10T02:22:54,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20efab466175d290 with lease ID 0x6eb1f02a33da5379: from storage DS-b00b0852-8eb8-4c51-b3c3-96c4bf5490a9 node DatanodeRegistration(127.0.0.1:45883, datanodeUuid=850558d8-1678-4950-8047-bc393b287634, infoPort=42487, infoSecurePort=0, ipcPort=41361, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:22:54,799 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfbede119ba09f8ed with lease ID 0x6eb1f02a33da537a: Processing first storage report for DS-00074578-f841-4c53-8b4e-923eda03c6b2 from datanode DatanodeRegistration(127.0.0.1:43857, datanodeUuid=39d3d08c-b056-404d-98b0-85dd3ad62ccf, infoPort=46657, infoSecurePort=0, ipcPort=33459, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889)
2024-12-10T02:22:54,799 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfbede119ba09f8ed with lease ID 0x6eb1f02a33da537a: from storage DS-00074578-f841-4c53-8b4e-923eda03c6b2 node DatanodeRegistration(127.0.0.1:43857, datanodeUuid=39d3d08c-b056-404d-98b0-85dd3ad62ccf, infoPort=46657, infoSecurePort=0, ipcPort=33459, storageInfo=lv=-57;cid=testClusterID;nsid=1201420063;c=1733797372889), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:22:54,893 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b
2024-12-10T02:22:54,971 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/zookeeper_0, clientPort=54862, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:22:54,982 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54862
2024-12-10T02:22:54,992 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:54,995 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:55,259 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:22:55,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:22:55,666 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0 with version=8
2024-12-10T02:22:55,667 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:22:55,761 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-10T02:22:56,014 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:22:56,026 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,026 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,031 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:22:56,031 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,031 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:22:56,175 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:22:56,239 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-10T02:22:56,247 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-10T02:22:56,252 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:22:56,279 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29888 (auto-detected)
2024-12-10T02:22:56,280 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-10T02:22:56,299 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38701
2024-12-10T02:22:56,320 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38701 connecting to ZooKeeper ensemble=127.0.0.1:54862
2024-12-10T02:22:56,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:387010x0, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:22:56,357 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38701-0x1019a2e56780000 connected
2024-12-10T02:22:56,388 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:56,391 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:56,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:22:56,409 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0, hbase.cluster.distributed=false
2024-12-10T02:22:56,441 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:22:56,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38701
2024-12-10T02:22:56,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38701
2024-12-10T02:22:56,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38701
2024-12-10T02:22:56,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38701
2024-12-10T02:22:56,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38701
2024-12-10T02:22:56,570 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:22:56,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,572 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,573 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:22:56,573 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:22:56,573 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:22:56,576 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:22:56,578 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:22:56,579 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37787
2024-12-10T02:22:56,582 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37787 connecting to ZooKeeper ensemble=127.0.0.1:54862
2024-12-10T02:22:56,583 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:56,588 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:56,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377870x0, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:22:56,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37787-0x1019a2e56780001 connected
2024-12-10T02:22:56,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:22:56,605 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:22:56,613 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:22:56,615 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:22:56,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:22:56,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37787
2024-12-10T02:22:56,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37787
2024-12-10T02:22:56,622 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37787
2024-12-10T02:22:56,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37787
2024-12-10T02:22:56,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37787
2024-12-10T02:22:56,641 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:38701
2024-12-10T02:22:56,642 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,38701,1733797375815
2024-12-10T02:22:56,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:22:56,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:22:56,652 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,38701,1733797375815
2024-12-10T02:22:56,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:22:56,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:56,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:56,674 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:22:56,676 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,38701,1733797375815 from backup master directory
2024-12-10T02:22:56,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,38701,1733797375815
2024-12-10T02:22:56,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:22:56,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:22:56,680 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:22:56,680 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,38701,1733797375815
2024-12-10T02:22:56,683 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-10T02:22:56,684 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-10T02:22:56,741 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase.id] with ID: 4cbf007d-e7f2-4c5f-b07b-284c33055cba
2024-12-10T02:22:56,742 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/.tmp/hbase.id
2024-12-10T02:22:56,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:22:56,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:22:56,757 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/.tmp/hbase.id]:[hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase.id]
2024-12-10T02:22:56,800 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:56,805 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:22:56,824 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms.
2024-12-10T02:22:56,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:56,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:56,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:22:56,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:22:56,863 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:22:56,865 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:22:56,871 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:22:56,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:22:56,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:22:56,924 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store
2024-12-10T02:22:56,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:22:56,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:22:56,951 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-10T02:22:56,955 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:56,956 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:22:56,957 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:22:56,957 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:22:56,959 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:22:56,959 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:22:56,959 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:22:56,961 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797376956Disabling compacts and flushes for region at 1733797376956Disabling writes for close at 1733797376959 (+3 ms)Writing region close event to WAL at 1733797376959Closed at 1733797376959
2024-12-10T02:22:56,964 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/.initializing
2024-12-10T02:22:56,964 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/WALs/d9f49988d155,38701,1733797375815
2024-12-10T02:22:56,987 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C38701%2C1733797375815, suffix=, logDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/WALs/d9f49988d155,38701,1733797375815, archiveDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/oldWALs, maxLogs=10
2024-12-10T02:22:56,999 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C38701%2C1733797375815.1733797376993
2024-12-10T02:22:57,023 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/WALs/d9f49988d155,38701,1733797375815/d9f49988d155%2C38701%2C1733797375815.1733797376993
2024-12-10T02:22:57,035 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42487:42487),(127.0.0.1/127.0.0.1:46657:46657)]
2024-12-10T02:22:57,037 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:22:57,037 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:57,041 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,042 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,080 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:22:57,107 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:22:57,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:57,115 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:22:57,119 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:22:57,119 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,120 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:22:57,121 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:22:57,123 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:22:57,124 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,125 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:22:57,125 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:22:57,127 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:22:57,128 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,128 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:22:57,129 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,133 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,134 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,139 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,139 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,142 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:22:57,146 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:22:57,150 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:22:57,151 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=858079, jitterRate=0.09110468626022339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:22:57,157 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797377054Initializing all the Stores at 1733797377057 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797377057Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797377058 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797377058Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797377058Cleaning up temporary data from old regions at 1733797377139 (+81 ms)Region opened successfully at 1733797377157 (+18 ms)
2024-12-10T02:22:57,158 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:22:57,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@532336ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:22:57,225 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:22:57,237 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:22:57,237 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:22:57,240 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:22:57,241 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-12-10T02:22:57,246 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-12-10T02:22:57,246 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:22:57,272 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:22:57,281 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:22:57,283 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:22:57,286 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:22:57,287 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:22:57,289 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:22:57,290 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:22:57,294 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:22:57,295 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:22:57,296 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:22:57,299 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:22:57,315 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:22:57,316 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:22:57,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:22:57,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:22:57,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,323 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,38701,1733797375815, sessionid=0x1019a2e56780000, setting cluster-up flag (Was=false)
2024-12-10T02:22:57,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,344 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:22:57,346 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,38701,1733797375815
2024-12-10T02:22:57,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:57,359 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:22:57,361 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,38701,1733797375815
2024-12-10T02:22:57,368 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:22:57,428 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(746): ClusterId : 4cbf007d-e7f2-4c5f-b07b-284c33055cba
2024-12-10T02:22:57,431 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:22:57,436 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:22:57,436 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:22:57,439 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:22:57,440 DEBUG [RS:0;d9f49988d155:37787 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@142b3e6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:22:57,445 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:22:57,455 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:37787
2024-12-10T02:22:57,455 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:22:57,458 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:22:57,458 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:22:57,459 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:22:57,462 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,38701,1733797375815 with port=37787, startcode=1733797376529
2024-12-10T02:22:57,462 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:22:57,468 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,38701,1733797375815
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,475 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:22:57,476 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,478 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797407478
2024-12-10T02:22:57,478 DEBUG [RS:0;d9f49988d155:37787 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:22:57,479 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:22:57,480 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:22:57,481 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:22:57,481 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:22:57,484 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:22:57,484 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:22:57,485 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:22:57,485 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:22:57,485 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,488 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,489 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:22:57,489 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:22:57,490 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:22:57,490 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:22:57,492 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:22:57,493 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:22:57,496 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797377494,5,FailOnTimeoutGroup]
2024-12-10T02:22:57,497 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797377497,5,FailOnTimeoutGroup]
2024-12-10T02:22:57,498 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,498 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:22:57,499 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,499 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,507 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:22:57,508 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:22:57,509 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:22:57,510 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0
2024-12-10T02:22:57,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:22:57,530 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:22:57,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:57,539 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:22:57,543 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:22:57,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,544 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:57,545 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:22:57,547 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:22:57,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,548 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:57,548 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:22:57,551 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:22:57,551 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,552 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:57,552 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:22:57,554 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:22:57,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:57,555 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:57,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:22:57,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740
2024-12-10T02:22:57,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740
2024-12-10T02:22:57,560 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:22:57,560 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:22:57,560 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:22:57,561 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:22:57,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:22:57,567 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38701 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,567 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:22:57,568 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=828188, jitterRate=0.05309663712978363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:22:57,570 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38701 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797377534Initializing all the Stores at 1733797377537 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797377537Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797377539 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797377539Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797377539Cleaning up temporary data from old regions at 1733797377560 (+21 ms)Region opened successfully at 1733797377571 (+11 ms)
2024-12-10T02:22:57,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:22:57,571 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:22:57,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:22:57,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:22:57,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:22:57,573 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:22:57,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797377571Disabling compacts and flushes for region at 1733797377571Disabling writes for close at 1733797377571Writing region close event to WAL at 1733797377573 (+2 ms)Closed at 1733797377573
2024-12-10T02:22:57,577 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:22:57,577 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:22:57,585 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:22:57,587 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0
2024-12-10T02:22:57,587 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39613
2024-12-10T02:22:57,587 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:22:57,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:22:57,592 DEBUG [RS:0;d9f49988d155:37787 {}] zookeeper.ZKUtil(111): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,592 WARN  [RS:0;d9f49988d155:37787 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:22:57,593 INFO  [RS:0;d9f49988d155:37787 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:22:57,593 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,594 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:22:57,595 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,37787,1733797376529]
2024-12-10T02:22:57,598 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:22:57,619 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:22:57,631 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:22:57,636 INFO  [RS:0;d9f49988d155:37787 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:22:57,637 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,637 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:22:57,643 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:22:57,645 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,645 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,645 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,645 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,645 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,646 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,647 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,647 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:22:57,647 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:22:57,647 DEBUG [RS:0;d9f49988d155:37787 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:22:57,648 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,648 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,648 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,648 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,649 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,649 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,37787,1733797376529-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:22:57,667 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:22:57,669 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,37787,1733797376529-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,670 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,670 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.Replication(171): d9f49988d155,37787,1733797376529 started
2024-12-10T02:22:57,691 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:57,691 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,37787,1733797376529, RpcServer on d9f49988d155/172.17.0.2:37787, sessionid=0x1019a2e56780001
2024-12-10T02:22:57,692 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:22:57,692 DEBUG [RS:0;d9f49988d155:37787 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,693 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,37787,1733797376529'
2024-12-10T02:22:57,693 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:22:57,694 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:22:57,695 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:22:57,695 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:22:57,695 DEBUG [RS:0;d9f49988d155:37787 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,37787,1733797376529
2024-12-10T02:22:57,695 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,37787,1733797376529'
2024-12-10T02:22:57,695 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:22:57,696 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:22:57,696 DEBUG [RS:0;d9f49988d155:37787 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:22:57,697 INFO  [RS:0;d9f49988d155:37787 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:22:57,697 INFO  [RS:0;d9f49988d155:37787 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:22:57,749 WARN  [d9f49988d155:38701 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:22:57,805 INFO  [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C37787%2C1733797376529, suffix=, logDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529, archiveDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs, maxLogs=32
2024-12-10T02:22:57,808 INFO  [RS:0;d9f49988d155:37787 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797377807
2024-12-10T02:22:57,816 INFO  [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797377807
2024-12-10T02:22:57,820 DEBUG [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:22:58,001 DEBUG [d9f49988d155:38701 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:22:58,013 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,37787,1733797376529
2024-12-10T02:22:58,020 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,37787,1733797376529, state=OPENING
2024-12-10T02:22:58,027 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:22:58,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:58,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:22:58,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:22:58,029 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:22:58,030 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:22:58,033 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,37787,1733797376529}]
2024-12-10T02:22:58,210 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:22:58,214 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36593, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:22:58,225 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:22:58,226 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:22:58,230 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C37787%2C1733797376529.meta, suffix=.meta, logDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529, archiveDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs, maxLogs=32
2024-12-10T02:22:58,231 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.meta.1733797378231.meta
2024-12-10T02:22:58,239 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.meta.1733797378231.meta
2024-12-10T02:22:58,243 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:22:58,245 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:22:58,246 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:22:58,249 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:22:58,254 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:22:58,259 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:22:58,259 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:58,260 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:22:58,260 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:22:58,263 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:22:58,265 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:22:58,265 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:58,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:58,266 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:22:58,267 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:22:58,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:58,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:58,268 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:22:58,269 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:22:58,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:58,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:58,270 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:22:58,271 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:22:58,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:58,272 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:22:58,272 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:22:58,274 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740
2024-12-10T02:22:58,276 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740
2024-12-10T02:22:58,279 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:22:58,279 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:22:58,280 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:22:58,282 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:22:58,284 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848935, jitterRate=0.0794779360294342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:22:58,284 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:22:58,286 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797378260Writing region info on filesystem at 1733797378260Initializing all the Stores at 1733797378262 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797378263 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797378263Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797378263Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797378263Cleaning up temporary data from old regions at 1733797378279 (+16 ms)Running coprocessor post-open hooks at 1733797378284 (+5 ms)Region opened successfully at 1733797378286 (+2 ms)
2024-12-10T02:22:58,293 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797378201
2024-12-10T02:22:58,304 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:22:58,304 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:22:58,306 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,37787,1733797376529
2024-12-10T02:22:58,308 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,37787,1733797376529, state=OPEN
2024-12-10T02:22:58,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:22:58,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:22:58,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:22:58,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:22:58,313 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,37787,1733797376529
2024-12-10T02:22:58,318 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:22:58,318 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,37787,1733797376529 in 281 msec
2024-12-10T02:22:58,325 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:22:58,325 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 736 msec
2024-12-10T02:22:58,326 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:22:58,327 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:22:58,347 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:22:58,348 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,37787,1733797376529, seqNum=-1]
2024-12-10T02:22:58,376 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:22:58,377 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48811, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:22:58,397 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0000 sec
2024-12-10T02:22:58,397 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797378397, completionTime=-1
2024-12-10T02:22:58,400 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:22:58,400 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:22:58,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:22:58,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797438430
2024-12-10T02:22:58,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797498430
2024-12-10T02:22:58,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec
2024-12-10T02:22:58,433 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,434 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,434 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,435 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:38701, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,436 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,437 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,443 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:22:58,465 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.785sec
2024-12-10T02:22:58,467 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:22:58,468 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:22:58,469 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:22:58,469 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:22:58,470 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:22:58,470 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:22:58,471 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:22:58,479 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:22:58,480 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:22:58,480 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38701,1733797375815-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:22:58,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64202c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:22:58,541 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-10T02:22:58,541 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-10T02:22:58,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,38701,-1 for getting cluster id
2024-12-10T02:22:58,547 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:22:58,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4cbf007d-e7f2-4c5f-b07b-284c33055cba'
2024-12-10T02:22:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:22:58,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4cbf007d-e7f2-4c5f-b07b-284c33055cba"

2024-12-10T02:22:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4cdb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:22:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,38701,-1]
2024-12-10T02:22:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:22:58,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:22:58,567 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:22:58,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@717eca08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:22:58,571 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:22:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,37787,1733797376529, seqNum=-1]
2024-12-10T02:22:58,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:22:58,580 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:22:58,601 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,38701,1733797375815
2024-12-10T02:22:58,601 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:22:58,609 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:22:58,613 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T02:22:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d9f49988d155,38701,1733797375815
2024-12-10T02:22:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d39bea9
2024-12-10T02:22:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T02:22:58,624 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T02:22:58,626 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T02:22:58,626 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T02:22:58,630 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:22:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-12-10T02:22:58,640 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T02:22:58,642 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-12-10T02:22:58,643 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:58,646 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T02:22:58,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:22:58,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741835_1011 (size=389)
2024-12-10T02:22:58,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741835_1011 (size=389)
2024-12-10T02:22:59,118 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 90be1ceae244b8e47f7b11968a3a88f2, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0
2024-12-10T02:22:59,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741836_1012 (size=72)
2024-12-10T02:22:59,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741836_1012 (size=72)
2024-12-10T02:22:59,129 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:59,129 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 90be1ceae244b8e47f7b11968a3a88f2, disabling compactions & flushes
2024-12-10T02:22:59,129 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,129 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,130 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2. after waiting 0 ms
2024-12-10T02:22:59,130 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,130 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,130 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 90be1ceae244b8e47f7b11968a3a88f2:
Waiting for close lock at 1733797379129Disabling compacts and flushes for region at 1733797379129Disabling writes for close at 1733797379130 (+1 ms)Writing region close event to WAL at 1733797379130Closed at 1733797379130
2024-12-10T02:22:59,132 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T02:22:59,139 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1733797379133"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797379133"}]},"ts":"1733797379133"}
2024-12-10T02:22:59,145 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T02:22:59,147 INFO  [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T02:22:59,149 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797379147"}]},"ts":"1733797379147"}
2024-12-10T02:22:59,154 INFO  [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-12-10T02:22:59,156 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=90be1ceae244b8e47f7b11968a3a88f2, ASSIGN}]
2024-12-10T02:22:59,158 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=90be1ceae244b8e47f7b11968a3a88f2, ASSIGN
2024-12-10T02:22:59,160 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=90be1ceae244b8e47f7b11968a3a88f2, ASSIGN; state=OFFLINE, location=d9f49988d155,37787,1733797376529; forceNewPlan=false, retain=false
2024-12-10T02:22:59,311 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=90be1ceae244b8e47f7b11968a3a88f2, regionState=OPENING, regionLocation=d9f49988d155,37787,1733797376529
2024-12-10T02:22:59,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=90be1ceae244b8e47f7b11968a3a88f2, ASSIGN because future has completed
2024-12-10T02:22:59,316 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90be1ceae244b8e47f7b11968a3a88f2, server=d9f49988d155,37787,1733797376529}]
2024-12-10T02:22:59,477 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,478 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 90be1ceae244b8e47f7b11968a3a88f2, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:22:59,478 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,479 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:22:59,479 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,479 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,481 INFO  [StoreOpener-90be1ceae244b8e47f7b11968a3a88f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 90be1ceae244b8e47f7b11968a3a88f2 
2024-12-10T02:22:59,484 INFO  [StoreOpener-90be1ceae244b8e47f7b11968a3a88f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90be1ceae244b8e47f7b11968a3a88f2 columnFamilyName info
2024-12-10T02:22:59,484 DEBUG [StoreOpener-90be1ceae244b8e47f7b11968a3a88f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:22:59,485 INFO  [StoreOpener-90be1ceae244b8e47f7b11968a3a88f2-1 {}] regionserver.HStore(327): Store=90be1ceae244b8e47f7b11968a3a88f2/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:22:59,485 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,487 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,487 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,488 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,488 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,491 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,494 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:22:59,494 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 90be1ceae244b8e47f7b11968a3a88f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=840312, jitterRate=0.06851299107074738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:22:59,494 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:22:59,495 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 90be1ceae244b8e47f7b11968a3a88f2:
Running coprocessor pre-open hook at 1733797379479Writing region info on filesystem at 1733797379479Initializing all the Stores at 1733797379481 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797379481Cleaning up temporary data from old regions at 1733797379488 (+7 ms)Running coprocessor post-open hooks at 1733797379494 (+6 ms)Region opened successfully at 1733797379495 (+1 ms)
2024-12-10T02:22:59,497 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2., pid=6, masterSystemTime=1733797379470
2024-12-10T02:22:59,502 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=90be1ceae244b8e47f7b11968a3a88f2, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,37787,1733797376529
2024-12-10T02:22:59,502 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,503 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:22:59,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90be1ceae244b8e47f7b11968a3a88f2, server=d9f49988d155,37787,1733797376529 because future has completed
2024-12-10T02:22:59,513 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T02:22:59,513 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 90be1ceae244b8e47f7b11968a3a88f2, server=d9f49988d155,37787,1733797376529 in 193 msec
2024-12-10T02:22:59,517 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T02:22:59,517 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=90be1ceae244b8e47f7b11968a3a88f2, ASSIGN in 357 msec
2024-12-10T02:22:59,518 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T02:22:59,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797379518"}]},"ts":"1733797379518"}
2024-12-10T02:22:59,522 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-12-10T02:22:59,523 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T02:22:59,527 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 891 msec
2024-12-10T02:23:03,763 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-10T02:23:03,806 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:23:03,808 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-12-10T02:23:06,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:23:06,236 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T02:23:06,237 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-10T02:23:06,237 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T02:23:06,238 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:23:06,238 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T02:23:06,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-12-10T02:23:06,239 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-12-10T02:23:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38701 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:23:08,706 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-12-10T02:23:08,709 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-12-10T02:23:08,715 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-12-10T02:23:08,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:23:08,716 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797388716
2024-12-10T02:23:08,724 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:08,725 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:08,725 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:08,725 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:08,725 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:08,725 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797377807 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797388716
2024-12-10T02:23:08,727 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:23:08,727 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797377807 is not closed yet, will try archiving it next time
2024-12-10T02:23:08,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741833_1009 (size=451)
2024-12-10T02:23:08,729 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741833_1009 (size=451)
2024-12-10T02:23:08,730 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797377807 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797377807
2024-12-10T02:23:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2., hostname=d9f49988d155,37787,1733797376529, seqNum=2]
2024-12-10T02:23:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:23:20,770 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90be1ceae244b8e47f7b11968a3a88f2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:23:20,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/d98588d2dc5b4911947f34e9a971a80f is 1080, key is row0001/info:/1733797388739/Put/seqid=0
2024-12-10T02:23:20,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741838_1014 (size=12509)
2024-12-10T02:23:20,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741838_1014 (size=12509)
2024-12-10T02:23:20,845 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/d98588d2dc5b4911947f34e9a971a80f
2024-12-10T02:23:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/d98588d2dc5b4911947f34e9a971a80f as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f
2024-12-10T02:23:20,905 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f, entries=7, sequenceid=11, filesize=12.2 K
2024-12-10T02:23:20,915 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 141ms, sequenceid=11, compaction requested=false
2024-12-10T02:23:20,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90be1ceae244b8e47f7b11968a3a88f2:

2024-12-10T02:23:24,890 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:23:28,780 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797408779
2024-12-10T02:23:28,988 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:28,988 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:28,988 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:28,988 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:28,989 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:28,989 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:28,989 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797388716 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797408779
2024-12-10T02:23:28,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:23:28,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797388716 is not closed yet, will try archiving it next time
2024-12-10T02:23:28,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741837_1013 (size=12399)
2024-12-10T02:23:28,992 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741837_1013 (size=12399)
2024-12-10T02:23:29,194 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:31,398 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:33,602 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:35,806 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:23:35,806 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90be1ceae244b8e47f7b11968a3a88f2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:23:36,008 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:36,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/9909c75d46f94875acddbd84735677c7 is 1080, key is row0008/info:/1733797402769/Put/seqid=0
2024-12-10T02:23:36,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741840_1016 (size=12509)
2024-12-10T02:23:36,023 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741840_1016 (size=12509)
2024-12-10T02:23:36,024 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/9909c75d46f94875acddbd84735677c7
2024-12-10T02:23:36,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/9909c75d46f94875acddbd84735677c7 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7
2024-12-10T02:23:36,047 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7, entries=7, sequenceid=21, filesize=12.2 K
2024-12-10T02:23:36,249 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:36,249 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 443ms, sequenceid=21, compaction requested=false
2024-12-10T02:23:36,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90be1ceae244b8e47f7b11968a3a88f2:

2024-12-10T02:23:36,250 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-12-10T02:23:36,250 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:23:36,250 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f because midkey is the same as first or last row
2024-12-10T02:23:38,011 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:38,482 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T02:23:38,482 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T02:23:40,215 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:40,217 WARN  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:40,218 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C37787%2C1733797376529:(num 1733797408779) roll requested
2024-12-10T02:23:40,218 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797420218
2024-12-10T02:23:40,426 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:40,426 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:40,427 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:40,427 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:40,427 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:40,427 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:40,427 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797408779 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797420218
2024-12-10T02:23:40,428 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:23:40,428 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797408779 is not closed yet, will try archiving it next time
2024-12-10T02:23:40,429 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797388716 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797388716
2024-12-10T02:23:40,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741839_1015 (size=7739)
2024-12-10T02:23:40,430 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741839_1015 (size=7739)
2024-12-10T02:23:42,419 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:44,479 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 90be1ceae244b8e47f7b11968a3a88f2, had cached 0 bytes from a total of 25018
2024-12-10T02:23:44,623 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:46,827 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:49,031 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:51,033 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T02:23:51,034 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797431033
2024-12-10T02:23:54,890 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:23:56,042 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:56,044 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK], DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK]]
2024-12-10T02:23:56,045 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C37787%2C1733797376529:(num 1733797431033) roll requested
2024-12-10T02:23:56,045 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:56,045 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:56,045 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:56,045 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:56,045 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:23:56,046 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797420218 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797431033
2024-12-10T02:23:56,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741841_1017 (size=4753)
2024-12-10T02:23:56,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741841_1017 (size=4753)
2024-12-10T02:23:56,049 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42487:42487),(127.0.0.1/127.0.0.1:46657:46657)]
2024-12-10T02:23:56,049 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797420218 is not closed yet, will try archiving it next time
2024-12-10T02:23:56,049 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797436049
2024-12-10T02:24:01,052 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:01,052 WARN  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:24:01,053 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90be1ceae244b8e47f7b11968a3a88f2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:24:01,058 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:01,058 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:03,054 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T02:24:06,055 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:06,055 WARN  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:06,055 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:06,055 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:06,056 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:06,056 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:06,056 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:06,056 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797431033 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797436049
2024-12-10T02:24:06,057 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42487:42487),(127.0.0.1/127.0.0.1:46657:46657)]
2024-12-10T02:24:06,057 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797431033 is not closed yet, will try archiving it next time
2024-12-10T02:24:06,057 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C37787%2C1733797376529:(num 1733797436049) roll requested
2024-12-10T02:24:06,058 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797446057
2024-12-10T02:24:06,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741842_1018 (size=1569)
2024-12-10T02:24:06,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741842_1018 (size=1569)
2024-12-10T02:24:06,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/c6006ef2ea8a431aa07f6d25a91a65e3 is 1080, key is row0015/info:/1733797417809/Put/seqid=0
2024-12-10T02:24:06,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741844_1020 (size=12509)
2024-12-10T02:24:06,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741844_1020 (size=12509)
2024-12-10T02:24:06,068 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/c6006ef2ea8a431aa07f6d25a91a65e3
2024-12-10T02:24:06,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/c6006ef2ea8a431aa07f6d25a91a65e3 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3
2024-12-10T02:24:06,087 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3, entries=7, sequenceid=31, filesize=12.2 K
2024-12-10T02:24:11,065 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:11,065 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:11,088 INFO  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:11,089 WARN  [FSHLog-0-hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0-prefix:d9f49988d155,37787,1733797376529 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45883,DS-66884d04-c986-46c8-adb8-6c58ee6d7e27,DISK], DatanodeInfoWithStorage[127.0.0.1:43857,DS-188419ca-0af3-46bd-a04b-5e5cbbe5a2b3,DISK]]
2024-12-10T02:24:11,089 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 10035ms, sequenceid=31, compaction requested=true
2024-12-10T02:24:11,089 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90be1ceae244b8e47f7b11968a3a88f2:

2024-12-10T02:24:11,089 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-12-10T02:24:11,089 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:11,089 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f because midkey is the same as first or last row
2024-12-10T02:24:11,089 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,089 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797436049 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797446057
2024-12-10T02:24:11,090 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:24:11,090 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797436049 is not closed yet, will try archiving it next time
2024-12-10T02:24:11,090 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797408779 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797408779
2024-12-10T02:24:11,091 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C37787%2C1733797376529:(num 1733797451091) roll requested
2024-12-10T02:24:11,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 90be1ceae244b8e47f7b11968a3a88f2:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:24:11,091 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797451091
2024-12-10T02:24:11,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741843_1019 (size=438)
2024-12-10T02:24:11,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741843_1019 (size=438)
2024-12-10T02:24:11,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:11,093 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797420218 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797420218
2024-12-10T02:24:11,094 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:24:11,095 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797431033 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797431033
2024-12-10T02:24:11,096 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:24:11,097 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797436049 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797436049
2024-12-10T02:24:11,098 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HStore(1541): 90be1ceae244b8e47f7b11968a3a88f2/info is initiating minor compaction (all files)
2024-12-10T02:24:11,098 INFO  [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 90be1ceae244b8e47f7b11968a3a88f2/info in TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:11,099 INFO  [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3] into tmpdir=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp, totalSize=36.6 K
2024-12-10T02:24:11,100 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting d98588d2dc5b4911947f34e9a971a80f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733797388739
2024-12-10T02:24:11,100 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9909c75d46f94875acddbd84735677c7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1733797402769
2024-12-10T02:24:11,101 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6006ef2ea8a431aa07f6d25a91a65e3, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1733797417809
2024-12-10T02:24:11,102 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,102 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,102 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,102 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,102 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,102 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797446057 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797451091
2024-12-10T02:24:11,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:24:11,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797446057 is not closed yet, will try archiving it next time
2024-12-10T02:24:11,104 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C37787%2C1733797376529.1733797451104
2024-12-10T02:24:11,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741845_1021 (size=93)
2024-12-10T02:24:11,105 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741845_1021 (size=93)
2024-12-10T02:24:11,106 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797446057 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs/d9f49988d155%2C37787%2C1733797376529.1733797446057
2024-12-10T02:24:11,118 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,118 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,118 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,118 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,118 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:11,119 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797451091 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797451104
2024-12-10T02:24:11,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741846_1022 (size=1258)
2024-12-10T02:24:11,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741846_1022 (size=1258)
2024-12-10T02:24:11,128 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46657:46657),(127.0.0.1/127.0.0.1:42487:42487)]
2024-12-10T02:24:11,129 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/WALs/d9f49988d155,37787,1733797376529/d9f49988d155%2C37787%2C1733797376529.1733797451091 is not closed yet, will try archiving it next time
2024-12-10T02:24:11,140 INFO  [RS:0;d9f49988d155:37787-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 90be1ceae244b8e47f7b11968a3a88f2#info#compaction#3 average throughput is 21.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:24:11,142 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/c7d8ca6bdc204bc4b4c8aa9898b20304 is 1080, key is row0001/info:/1733797388739/Put/seqid=0
2024-12-10T02:24:11,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741848_1024 (size=27710)
2024-12-10T02:24:11,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741848_1024 (size=27710)
2024-12-10T02:24:11,162 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/c7d8ca6bdc204bc4b4c8aa9898b20304 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c7d8ca6bdc204bc4b4c8aa9898b20304
2024-12-10T02:24:11,178 INFO  [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 90be1ceae244b8e47f7b11968a3a88f2/info of 90be1ceae244b8e47f7b11968a3a88f2 into c7d8ca6bdc204bc4b4c8aa9898b20304(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:24:11,179 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 90be1ceae244b8e47f7b11968a3a88f2:

2024-12-10T02:24:11,180 INFO  [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2., storeName=90be1ceae244b8e47f7b11968a3a88f2/info, priority=13, startTime=1733797451090; duration=0sec
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c7d8ca6bdc204bc4b4c8aa9898b20304 because midkey is the same as first or last row
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c7d8ca6bdc204bc4b4c8aa9898b20304 because midkey is the same as first or last row
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c7d8ca6bdc204bc4b4c8aa9898b20304 because midkey is the same as first or last row
2024-12-10T02:24:11,181 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:11,182 DEBUG [RS:0;d9f49988d155:37787-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 90be1ceae244b8e47f7b11968a3a88f2:info
2024-12-10T02:24:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37787 {}] regionserver.HRegion(8855): Flush requested on 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:24:23,128 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 90be1ceae244b8e47f7b11968a3a88f2 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:24:23,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/33e8497cac0e4aa1b8ee605e4eb1e0bc is 1080, key is row0022/info:/1733797451106/Put/seqid=0
2024-12-10T02:24:23,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741849_1025 (size=12509)
2024-12-10T02:24:23,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741849_1025 (size=12509)
2024-12-10T02:24:23,143 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/33e8497cac0e4aa1b8ee605e4eb1e0bc
2024-12-10T02:24:23,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/33e8497cac0e4aa1b8ee605e4eb1e0bc as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/33e8497cac0e4aa1b8ee605e4eb1e0bc
2024-12-10T02:24:23,160 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/33e8497cac0e4aa1b8ee605e4eb1e0bc, entries=7, sequenceid=42, filesize=12.2 K
2024-12-10T02:24:23,161 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 34ms, sequenceid=42, compaction requested=false
2024-12-10T02:24:23,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 90be1ceae244b8e47f7b11968a3a88f2:

2024-12-10T02:24:23,161 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-12-10T02:24:23,161 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:23,161 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c7d8ca6bdc204bc4b4c8aa9898b20304 because midkey is the same as first or last row
2024-12-10T02:24:24,890 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:24:29,479 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 90be1ceae244b8e47f7b11968a3a88f2, had cached 0 bytes from a total of 40219
2024-12-10T02:24:31,138 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:24:31,139 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:24:31,139 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:31,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:31,147 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:31,147 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:24:31,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:24:31,147 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=278837223, stopped=false
2024-12-10T02:24:31,147 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,38701,1733797375815
2024-12-10T02:24:31,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:31,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:31,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:31,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:31,150 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:24:31,150 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:24:31,150 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:31,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:31,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:31,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:31,151 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,37787,1733797376529' *****
2024-12-10T02:24:31,151 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:24:31,151 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:24:31,151 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:24:31,151 INFO  [RS:0;d9f49988d155:37787 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:24:31,152 INFO  [RS:0;d9f49988d155:37787 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:24:31,152 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(3091): Received CLOSE for 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,37787,1733797376529
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:37787.
2024-12-10T02:24:31,153 DEBUG [RS:0;d9f49988d155:37787 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:31,153 DEBUG [RS:0;d9f49988d155:37787 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:24:31,153 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 90be1ceae244b8e47f7b11968a3a88f2, disabling compactions & flushes
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:24:31,153 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:31,153 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:31,153 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:24:31,153 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2. after waiting 0 ms
2024-12-10T02:24:31,153 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:31,153 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 90be1ceae244b8e47f7b11968a3a88f2 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-12-10T02:24:31,154 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T02:24:31,154 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1325): Online Regions={90be1ceae244b8e47f7b11968a3a88f2=TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2., 1588230740=hbase:meta,,1.1588230740}
2024-12-10T02:24:31,154 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:24:31,154 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:24:31,154 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:24:31,154 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:24:31,154 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:24:31,154 DEBUG [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 90be1ceae244b8e47f7b11968a3a88f2
2024-12-10T02:24:31,154 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-12-10T02:24:31,159 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/8e095dafbb784b889bcb336b0f62679f is 1080, key is row0029/info:/1733797465129/Put/seqid=0
2024-12-10T02:24:31,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741850_1026 (size=8193)
2024-12-10T02:24:31,167 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741850_1026 (size=8193)
2024-12-10T02:24:31,167 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/8e095dafbb784b889bcb336b0f62679f
2024-12-10T02:24:31,176 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/info/8d6db21ca9f44bdfa408de894de454d5 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2./info:regioninfo/1733797379502/Put/seqid=0
2024-12-10T02:24:31,176 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/.tmp/info/8e095dafbb784b889bcb336b0f62679f as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/8e095dafbb784b889bcb336b0f62679f
2024-12-10T02:24:31,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741851_1027 (size=7016)
2024-12-10T02:24:31,183 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741851_1027 (size=7016)
2024-12-10T02:24:31,183 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/info/8d6db21ca9f44bdfa408de894de454d5
2024-12-10T02:24:31,185 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/8e095dafbb784b889bcb336b0f62679f, entries=3, sequenceid=48, filesize=8.0 K
2024-12-10T02:24:31,186 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 33ms, sequenceid=48, compaction requested=true
2024-12-10T02:24:31,187 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3] to archive
2024-12-10T02:24:31,190 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:24:31,195 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/archive/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/d98588d2dc5b4911947f34e9a971a80f
2024-12-10T02:24:31,196 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/archive/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/9909c75d46f94875acddbd84735677c7
2024-12-10T02:24:31,196 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3 to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/archive/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/info/c6006ef2ea8a431aa07f6d25a91a65e3
2024-12-10T02:24:31,214 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/ns/690c3f82ffc14d5194436c624c5b4d12 is 43, key is default/ns:d/1733797378382/Put/seqid=0
2024-12-10T02:24:31,212 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d9f49988d155:38701 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T02:24:31,217 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d98588d2dc5b4911947f34e9a971a80f=12509, 9909c75d46f94875acddbd84735677c7=12509, c6006ef2ea8a431aa07f6d25a91a65e3=12509]
2024-12-10T02:24:31,222 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741852_1028 (size=5153)
2024-12-10T02:24:31,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741852_1028 (size=5153)
2024-12-10T02:24:31,223 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/ns/690c3f82ffc14d5194436c624c5b4d12
2024-12-10T02:24:31,225 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/default/TestLogRolling-testSlowSyncLogRolling/90be1ceae244b8e47f7b11968a3a88f2/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-12-10T02:24:31,227 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:31,228 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 90be1ceae244b8e47f7b11968a3a88f2:
Waiting for close lock at 1733797471153Running coprocessor pre-close hooks at 1733797471153Disabling compacts and flushes for region at 1733797471153Disabling writes for close at 1733797471153Obtaining lock to block concurrent updates at 1733797471153Preparing flush snapshotting stores in 90be1ceae244b8e47f7b11968a3a88f2 at 1733797471153Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1733797471154 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2. at 1733797471155 (+1 ms)Flushing 90be1ceae244b8e47f7b11968a3a88f2/info: creating writer at 1733797471155Flushing 90be1ceae244b8e47f7b11968a3a88f2/info: appending metadata at 1733797471159 (+4 ms)Flushing 90be1ceae244b8e47f7b11968a3a88f2/info: closing flushed file at 1733797471159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f664217: reopening flushed file at 1733797471175 (+16 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 90be1ceae244b8e47f7b11968a3a88f2 in 33ms, sequenceid=48, compaction requested=true at 1733797471186 (+11 ms)Writing region close event to WAL at 1733797471219 (+33 ms)Running coprocessor post-close hooks at 1733797471225 (+6 ms)Closed at 1733797471227 (+2 ms)
2024-12-10T02:24:31,228 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1733797378626.90be1ceae244b8e47f7b11968a3a88f2.
2024-12-10T02:24:31,250 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/table/b50ac8088d5349e391e27a05303af5ba is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1733797379518/Put/seqid=0
2024-12-10T02:24:31,257 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741853_1029 (size=5396)
2024-12-10T02:24:31,257 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741853_1029 (size=5396)
2024-12-10T02:24:31,257 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/table/b50ac8088d5349e391e27a05303af5ba
2024-12-10T02:24:31,266 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/info/8d6db21ca9f44bdfa408de894de454d5 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/info/8d6db21ca9f44bdfa408de894de454d5
2024-12-10T02:24:31,274 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/info/8d6db21ca9f44bdfa408de894de454d5, entries=10, sequenceid=11, filesize=6.9 K
2024-12-10T02:24:31,276 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/ns/690c3f82ffc14d5194436c624c5b4d12 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/ns/690c3f82ffc14d5194436c624c5b4d12
2024-12-10T02:24:31,283 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/ns/690c3f82ffc14d5194436c624c5b4d12, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T02:24:31,284 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/.tmp/table/b50ac8088d5349e391e27a05303af5ba as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/table/b50ac8088d5349e391e27a05303af5ba
2024-12-10T02:24:31,292 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/table/b50ac8088d5349e391e27a05303af5ba, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T02:24:31,293 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 139ms, sequenceid=11, compaction requested=false
2024-12-10T02:24:31,299 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T02:24:31,300 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:24:31,300 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:31,300 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797471154Running coprocessor pre-close hooks at 1733797471154Disabling compacts and flushes for region at 1733797471154Disabling writes for close at 1733797471154Obtaining lock to block concurrent updates at 1733797471154Preparing flush snapshotting stores in 1588230740 at 1733797471154Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1733797471154Flushing stores of hbase:meta,,1.1588230740 at 1733797471156 (+2 ms)Flushing 1588230740/info: creating writer at 1733797471156Flushing 1588230740/info: appending metadata at 1733797471175 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733797471175Flushing 1588230740/ns: creating writer at 1733797471191 (+16 ms)Flushing 1588230740/ns: appending metadata at 1733797471214 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1733797471214Flushing 1588230740/table: creating writer at 1733797471232 (+18 ms)Flushing 1588230740/table: appending metadata at 1733797471250 (+18 ms)Flushing 1588230740/table: closing flushed file at 1733797471250Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37441d9a: reopening flushed file at 1733797471265 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2012d047: reopening flushed file at 1733797471275 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ff18e8: reopening flushed file at 1733797471284 (+9 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 139ms, sequenceid=11, compaction requested=false at 1733797471293 (+9 ms)Writing region close event to WAL at 1733797471295 (+2 ms)Running coprocessor post-close hooks at 1733797471300 (+5 ms)Closed at 1733797471300
2024-12-10T02:24:31,300 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:31,354 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,37787,1733797376529; all regions closed.
2024-12-10T02:24:31,356 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,356 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,356 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,356 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,356 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741834_1010 (size=3066)
2024-12-10T02:24:31,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741834_1010 (size=3066)
2024-12-10T02:24:31,363 DEBUG [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs
2024-12-10T02:24:31,363 INFO  [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C37787%2C1733797376529.meta:.meta(num 1733797378231)
2024-12-10T02:24:31,364 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,364 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,364 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,364 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,364 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,366 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741847_1023 (size=12695)
2024-12-10T02:24:31,367 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741847_1023 (size=12695)
2024-12-10T02:24:31,370 DEBUG [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/oldWALs
2024-12-10T02:24:31,370 INFO  [RS:0;d9f49988d155:37787 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C37787%2C1733797376529:(num 1733797451104)
2024-12-10T02:24:31,370 DEBUG [RS:0;d9f49988d155:37787 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:31,370 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:24:31,370 INFO  [RS:0;d9f49988d155:37787 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:24:31,370 INFO  [RS:0;d9f49988d155:37787 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T02:24:31,370 INFO  [RS:0;d9f49988d155:37787 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:24:31,371 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:24:31,371 INFO  [RS:0;d9f49988d155:37787 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37787
2024-12-10T02:24:31,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,37787,1733797376529
2024-12-10T02:24:31,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:24:31,375 INFO  [RS:0;d9f49988d155:37787 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:24:31,378 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,37787,1733797376529]
2024-12-10T02:24:31,381 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,37787,1733797376529 already deleted, retry=false
2024-12-10T02:24:31,381 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,37787,1733797376529 expired; onlineServers=0
2024-12-10T02:24:31,381 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,38701,1733797375815' *****
2024-12-10T02:24:31,381 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:24:31,381 INFO  [M:0;d9f49988d155:38701 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:24:31,381 INFO  [M:0;d9f49988d155:38701 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:24:31,382 DEBUG [M:0;d9f49988d155:38701 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:24:31,382 DEBUG [M:0;d9f49988d155:38701 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:24:31,382 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:24:31,382 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797377497 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797377497,5,FailOnTimeoutGroup]
2024-12-10T02:24:31,382 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797377494 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797377494,5,FailOnTimeoutGroup]
2024-12-10T02:24:31,382 INFO  [M:0;d9f49988d155:38701 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:24:31,382 INFO  [M:0;d9f49988d155:38701 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:24:31,382 DEBUG [M:0;d9f49988d155:38701 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:24:31,382 INFO  [M:0;d9f49988d155:38701 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:24:31,382 INFO  [M:0;d9f49988d155:38701 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:24:31,383 INFO  [M:0;d9f49988d155:38701 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:24:31,383 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:24:31,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:24:31,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:31,384 DEBUG [M:0;d9f49988d155:38701 {}] zookeeper.ZKUtil(347): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:24:31,384 WARN  [M:0;d9f49988d155:38701 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:24:31,385 INFO  [M:0;d9f49988d155:38701 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/.lastflushedseqids
2024-12-10T02:24:31,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741854_1030 (size=130)
2024-12-10T02:24:31,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741854_1030 (size=130)
2024-12-10T02:24:31,397 INFO  [M:0;d9f49988d155:38701 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:24:31,397 INFO  [M:0;d9f49988d155:38701 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:24:31,397 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:24:31,397 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:31,397 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:31,397 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:24:31,397 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:31,397 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-12-10T02:24:31,415 DEBUG [M:0;d9f49988d155:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5642e1508c5c4fbe81483f0c0972200d is 82, key is hbase:meta,,1/info:regioninfo/1733797378305/Put/seqid=0
2024-12-10T02:24:31,421 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741855_1031 (size=5672)
2024-12-10T02:24:31,422 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741855_1031 (size=5672)
2024-12-10T02:24:31,422 INFO  [M:0;d9f49988d155:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5642e1508c5c4fbe81483f0c0972200d
2024-12-10T02:24:31,445 DEBUG [M:0;d9f49988d155:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/681d4fc7f6404595a0a47d053665234a is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733797379526/Put/seqid=0
2024-12-10T02:24:31,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741856_1032 (size=6247)
2024-12-10T02:24:31,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741856_1032 (size=6247)
2024-12-10T02:24:31,451 INFO  [M:0;d9f49988d155:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/681d4fc7f6404595a0a47d053665234a
2024-12-10T02:24:31,458 INFO  [M:0;d9f49988d155:38701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 681d4fc7f6404595a0a47d053665234a
2024-12-10T02:24:31,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:31,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37787-0x1019a2e56780001, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:31,478 INFO  [RS:0;d9f49988d155:37787 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:24:31,478 INFO  [RS:0;d9f49988d155:37787 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,37787,1733797376529; zookeeper connection closed.
2024-12-10T02:24:31,479 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@408ffaba {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@408ffaba
2024-12-10T02:24:31,479 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:24:31,480 DEBUG [M:0;d9f49988d155:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b65468b14149bd849adfc61b686261 is 69, key is d9f49988d155,37787,1733797376529/rs:state/1733797377572/Put/seqid=0
2024-12-10T02:24:31,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741857_1033 (size=5156)
2024-12-10T02:24:31,486 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741857_1033 (size=5156)
2024-12-10T02:24:31,486 INFO  [M:0;d9f49988d155:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b65468b14149bd849adfc61b686261
2024-12-10T02:24:31,507 DEBUG [M:0;d9f49988d155:38701 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4759b688b4340718326862aac10fc81 is 52, key is load_balancer_on/state:d/1733797378605/Put/seqid=0
2024-12-10T02:24:31,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741858_1034 (size=5056)
2024-12-10T02:24:31,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741858_1034 (size=5056)
2024-12-10T02:24:31,515 INFO  [M:0;d9f49988d155:38701 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4759b688b4340718326862aac10fc81
2024-12-10T02:24:31,523 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5642e1508c5c4fbe81483f0c0972200d as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5642e1508c5c4fbe81483f0c0972200d
2024-12-10T02:24:31,530 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5642e1508c5c4fbe81483f0c0972200d, entries=8, sequenceid=59, filesize=5.5 K
2024-12-10T02:24:31,531 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/681d4fc7f6404595a0a47d053665234a as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/681d4fc7f6404595a0a47d053665234a
2024-12-10T02:24:31,538 INFO  [M:0;d9f49988d155:38701 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 681d4fc7f6404595a0a47d053665234a
2024-12-10T02:24:31,538 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/681d4fc7f6404595a0a47d053665234a, entries=6, sequenceid=59, filesize=6.1 K
2024-12-10T02:24:31,539 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/69b65468b14149bd849adfc61b686261 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69b65468b14149bd849adfc61b686261
2024-12-10T02:24:31,545 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/69b65468b14149bd849adfc61b686261, entries=1, sequenceid=59, filesize=5.0 K
2024-12-10T02:24:31,546 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4759b688b4340718326862aac10fc81 as hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4759b688b4340718326862aac10fc81
2024-12-10T02:24:31,552 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4759b688b4340718326862aac10fc81, entries=1, sequenceid=59, filesize=4.9 K
2024-12-10T02:24:31,553 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=59, compaction requested=false
2024-12-10T02:24:31,555 INFO  [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:31,555 DEBUG [M:0;d9f49988d155:38701 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797471397Disabling compacts and flushes for region at 1733797471397Disabling writes for close at 1733797471397Obtaining lock to block concurrent updates at 1733797471397Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797471397Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1733797471398 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797471399 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797471399Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797471414 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797471414Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797471429 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797471444 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797471444Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797471458 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797471479 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797471479Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797471492 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797471507 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797471507Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cfc3da2: reopening flushed file at 1733797471522 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3fc0b0: reopening flushed file at 1733797471530 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b0ff6d: reopening flushed file at 1733797471538 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bdad229: reopening flushed file at 1733797471545 (+7 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=59, compaction requested=false at 1733797471553 (+8 ms)Writing region close event to WAL at 1733797471555 (+2 ms)Closed at 1733797471555
2024-12-10T02:24:31,556 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,556 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,556 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,556 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,556 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:31,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43857 is added to blk_1073741830_1006 (size=27973)
2024-12-10T02:24:31,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45883 is added to blk_1073741830_1006 (size=27973)
2024-12-10T02:24:31,559 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:24:31,560 INFO  [M:0;d9f49988d155:38701 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:24:31,560 INFO  [M:0;d9f49988d155:38701 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38701
2024-12-10T02:24:31,560 INFO  [M:0;d9f49988d155:38701 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:24:31,653 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:24:31,665 INFO  [M:0;d9f49988d155:38701 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:24:31,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:31,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38701-0x1019a2e56780000, quorum=127.0.0.1:54862, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:31,669 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:31,672 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:31,672 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:31,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:31,672 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:31,675 WARN  [BP-754960761-172.17.0.2-1733797372889 heartbeating to localhost/127.0.0.1:39613 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:31,675 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:31,675 WARN  [BP-754960761-172.17.0.2-1733797372889 heartbeating to localhost/127.0.0.1:39613 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-754960761-172.17.0.2-1733797372889 (Datanode Uuid 850558d8-1678-4950-8047-bc393b287634) service to localhost/127.0.0.1:39613
2024-12-10T02:24:31,675 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:31,677 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data3/current/BP-754960761-172.17.0.2-1733797372889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:31,677 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data4/current/BP-754960761-172.17.0.2-1733797372889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:31,677 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:31,679 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:31,680 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:31,680 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:31,680 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:31,680 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:31,682 WARN  [BP-754960761-172.17.0.2-1733797372889 heartbeating to localhost/127.0.0.1:39613 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:31,682 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:31,682 WARN  [BP-754960761-172.17.0.2-1733797372889 heartbeating to localhost/127.0.0.1:39613 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-754960761-172.17.0.2-1733797372889 (Datanode Uuid 39d3d08c-b056-404d-98b0-85dd3ad62ccf) service to localhost/127.0.0.1:39613
2024-12-10T02:24:31,682 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:31,682 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data1/current/BP-754960761-172.17.0.2-1733797372889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:31,683 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/cluster_671b13eb-b66d-261d-5717-ea8ce5358ecf/data/data2/current/BP-754960761-172.17.0.2-1733797372889 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:31,683 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:31,693 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:24:31,693 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:31,694 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:31,694 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:31,694 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:31,703 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:24:31,738 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:24:31,749 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=83 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39613 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HFileArchiver-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39613
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: master/d9f49988d155:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39613 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@221b72a4
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:39613
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39613 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:39613
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/d9f49988d155:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39613
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39613
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/d9f49988d155:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=48 (was 160), ProcessCount=11 (was 11), AvailableMemoryMB=4372 (was 4939)
2024-12-10T02:24:31,756 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=84, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=48, ProcessCount=11, AvailableMemoryMB=4371
2024-12-10T02:24:31,756 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:24:31,756 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.log.dir so I do NOT create it in target/test-data/29d50935-2716-deae-0766-4e88be1add4f
2024-12-10T02:24:31,756 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7851f79-964d-cfa3-cba0-f0277b977f3b/hadoop.tmp.dir so I do NOT create it in target/test-data/29d50935-2716-deae-0766-4e88be1add4f
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6, deleteOnExit=true
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/test.cache.data in system properties and HBase conf
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:24:31,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:24:31,757 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:24:31,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:24:31,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:24:31,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:24:31,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:24:31,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:24:31,774 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:24:31,866 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:31,872 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:31,873 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:31,873 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:31,873 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:31,874 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:31,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3197ca45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:31,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45e3157d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:31,993 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@511dc70f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/java.io.tmpdir/jetty-localhost-40559-hadoop-hdfs-3_4_1-tests_jar-_-any-3989328696800702489/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:24:31,994 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e469283{HTTP/1.1, (http/1.1)}{localhost:40559}
2024-12-10T02:24:31,994 INFO  [Time-limited test {}] server.Server(415): Started @101052ms
2024-12-10T02:24:32,008 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:24:32,093 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:32,097 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:32,099 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:32,099 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:32,100 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:32,100 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4edee9ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:32,101 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@276f8783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:32,218 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5d4bdc00{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/java.io.tmpdir/jetty-localhost-39013-hadoop-hdfs-3_4_1-tests_jar-_-any-9823972566816442417/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:32,219 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@687b21ce{HTTP/1.1, (http/1.1)}{localhost:39013}
2024-12-10T02:24:32,219 INFO  [Time-limited test {}] server.Server(415): Started @101277ms
2024-12-10T02:24:32,221 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:32,260 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:32,265 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:32,267 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:32,267 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:32,267 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:32,267 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b44e274{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:32,268 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@376d199b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:32,342 WARN  [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data1/current/BP-2060907613-172.17.0.2-1733797471794/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:32,342 WARN  [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data2/current/BP-2060907613-172.17.0.2-1733797471794/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:32,364 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:32,367 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x45c7d366d9d17ac5 with lease ID 0x70ff7a2a4461ea6: Processing first storage report for DS-a2a8ca0b-e536-4ddb-aaba-754773c04908 from datanode DatanodeRegistration(127.0.0.1:46783, datanodeUuid=f5dfb3f2-c576-4c4f-b605-0362e9d32cd8, infoPort=36657, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794)
2024-12-10T02:24:32,367 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x45c7d366d9d17ac5 with lease ID 0x70ff7a2a4461ea6: from storage DS-a2a8ca0b-e536-4ddb-aaba-754773c04908 node DatanodeRegistration(127.0.0.1:46783, datanodeUuid=f5dfb3f2-c576-4c4f-b605-0362e9d32cd8, infoPort=36657, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:32,367 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x45c7d366d9d17ac5 with lease ID 0x70ff7a2a4461ea6: Processing first storage report for DS-cd448c30-be1d-4218-82d7-1cdb1090562d from datanode DatanodeRegistration(127.0.0.1:46783, datanodeUuid=f5dfb3f2-c576-4c4f-b605-0362e9d32cd8, infoPort=36657, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794)
2024-12-10T02:24:32,367 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x45c7d366d9d17ac5 with lease ID 0x70ff7a2a4461ea6: from storage DS-cd448c30-be1d-4218-82d7-1cdb1090562d node DatanodeRegistration(127.0.0.1:46783, datanodeUuid=f5dfb3f2-c576-4c4f-b605-0362e9d32cd8, infoPort=36657, infoSecurePort=0, ipcPort=38083, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:32,392 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@824b6ae{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/java.io.tmpdir/jetty-localhost-32915-hadoop-hdfs-3_4_1-tests_jar-_-any-12956038079181606616/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:32,392 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5d7e5c06{HTTP/1.1, (http/1.1)}{localhost:32915}
2024-12-10T02:24:32,392 INFO  [Time-limited test {}] server.Server(415): Started @101450ms
2024-12-10T02:24:32,394 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:32,501 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data3/current/BP-2060907613-172.17.0.2-1733797471794/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:32,501 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data4/current/BP-2060907613-172.17.0.2-1733797471794/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:32,519 WARN  [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:32,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51881e1ead134561 with lease ID 0x70ff7a2a4461ea7: Processing first storage report for DS-a47b092d-d591-4b52-8d9e-663da3dcf8c3 from datanode DatanodeRegistration(127.0.0.1:40155, datanodeUuid=30fcb61b-db82-4b05-90bd-2df1c208fe62, infoPort=34929, infoSecurePort=0, ipcPort=39711, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794)
2024-12-10T02:24:32,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51881e1ead134561 with lease ID 0x70ff7a2a4461ea7: from storage DS-a47b092d-d591-4b52-8d9e-663da3dcf8c3 node DatanodeRegistration(127.0.0.1:40155, datanodeUuid=30fcb61b-db82-4b05-90bd-2df1c208fe62, infoPort=34929, infoSecurePort=0, ipcPort=39711, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:32,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51881e1ead134561 with lease ID 0x70ff7a2a4461ea7: Processing first storage report for DS-c94a92d9-7fcb-4b51-adb6-a64c609823e8 from datanode DatanodeRegistration(127.0.0.1:40155, datanodeUuid=30fcb61b-db82-4b05-90bd-2df1c208fe62, infoPort=34929, infoSecurePort=0, ipcPort=39711, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794)
2024-12-10T02:24:32,522 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51881e1ead134561 with lease ID 0x70ff7a2a4461ea7: from storage DS-c94a92d9-7fcb-4b51-adb6-a64c609823e8 node DatanodeRegistration(127.0.0.1:40155, datanodeUuid=30fcb61b-db82-4b05-90bd-2df1c208fe62, infoPort=34929, infoSecurePort=0, ipcPort=39711, storageInfo=lv=-57;cid=testClusterID;nsid=602503049;c=1733797471794), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:32,623 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f
2024-12-10T02:24:32,626 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/zookeeper_0, clientPort=52464, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:24:32,627 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52464
2024-12-10T02:24:32,628 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,629 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:24:32,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:24:32,643 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b with version=8
2024-12-10T02:24:32,643 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,645 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:24:32,646 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:24:32,646 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:24:32,646 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36973
2024-12-10T02:24:32,648 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36973 connecting to ZooKeeper ensemble=127.0.0.1:52464
2024-12-10T02:24:32,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369730x0, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:24:32,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36973-0x1019a2fd4010000 connected
2024-12-10T02:24:32,686 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,688 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:32,691 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b, hbase.cluster.distributed=false
2024-12-10T02:24:32,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:24:32,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36973
2024-12-10T02:24:32,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36973
2024-12-10T02:24:32,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36973
2024-12-10T02:24:32,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36973
2024-12-10T02:24:32,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36973
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:24:32,712 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:24:32,713 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39891
2024-12-10T02:24:32,714 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39891 connecting to ZooKeeper ensemble=127.0.0.1:52464
2024-12-10T02:24:32,715 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,718 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398910x0, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:24:32,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39891-0x1019a2fd4010001 connected
2024-12-10T02:24:32,724 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:32,724 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:24:32,727 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:24:32,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:24:32,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:24:32,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39891
2024-12-10T02:24:32,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39891
2024-12-10T02:24:32,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39891
2024-12-10T02:24:32,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39891
2024-12-10T02:24:32,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39891
2024-12-10T02:24:32,756 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:36973
2024-12-10T02:24:32,757 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:32,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:32,760 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:24:32,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,764 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:24:32,765 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,36973,1733797472645 from backup master directory
2024-12-10T02:24:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:32,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:32,766 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:24:32,766 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,772 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/hbase.id] with ID: 7b6df85f-0950-4492-b666-ba1581539109
2024-12-10T02:24:32,772 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/.tmp/hbase.id
2024-12-10T02:24:32,781 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:24:32,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:24:32,782 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/.tmp/hbase.id]:[hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/hbase.id]
2024-12-10T02:24:32,797 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:32,798 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:24:32,799 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T02:24:32,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:24:32,810 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:24:32,810 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:24:32,811 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:24:32,812 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:32,820 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:24:32,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:24:32,821 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store
2024-12-10T02:24:32,828 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:24:32,829 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:24:32,829 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:32,829 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:24:32,829 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:32,829 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:32,829 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:24:32,829 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:32,829 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:32,830 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797472829Disabling compacts and flushes for region at 1733797472829Disabling writes for close at 1733797472829Writing region close event to WAL at 1733797472829Closed at 1733797472829
2024-12-10T02:24:32,831 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/.initializing
2024-12-10T02:24:32,831 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/WALs/d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,834 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C36973%2C1733797472645, suffix=, logDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/WALs/d9f49988d155,36973,1733797472645, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/oldWALs, maxLogs=10
2024-12-10T02:24:32,835 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C36973%2C1733797472645.1733797472834
2024-12-10T02:24:32,840 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/WALs/d9f49988d155,36973,1733797472645/d9f49988d155%2C36973%2C1733797472645.1733797472834
2024-12-10T02:24:32,843 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36657:36657),(127.0.0.1/127.0.0.1:34929:34929)]
2024-12-10T02:24:32,843 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:24:32,843 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:32,844 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,844 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,846 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:32,848 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:24:32,848 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,849 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:32,849 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:32,850 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:24:32,850 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,851 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:32,851 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:32,853 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:24:32,853 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,853 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:32,854 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:32,855 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:24:32,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,855 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:32,855 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,856 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,857 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,858 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,858 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,859 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:24:32,861 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:32,863 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:24:32,863 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=691990, jitterRate=-0.12009042501449585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:24:32,864 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797472844Initializing all the Stores at 1733797472845 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797472845Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797472846 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797472846Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797472846Cleaning up temporary data from old regions at 1733797472858 (+12 ms)Region opened successfully at 1733797472864 (+6 ms)
2024-12-10T02:24:32,865 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:24:32,869 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd6167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:24:32,870 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:24:32,870 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:24:32,870 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:24:32,870 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:24:32,871 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:24:32,871 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:24:32,871 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:24:32,874 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:24:32,874 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:24:32,876 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:24:32,876 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:24:32,877 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:24:32,880 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:24:32,881 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:24:32,882 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:24:32,883 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:24:32,884 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:24:32,885 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:24:32,887 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:24:32,889 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:24:32,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:32,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:32,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,893 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,36973,1733797472645, sessionid=0x1019a2fd4010000, setting cluster-up flag (Was=false)
2024-12-10T02:24:32,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,903 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:24:32,904 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:32,914 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:24:32,915 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,36973,1733797472645
2024-12-10T02:24:32,916 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:24:32,918 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:32,919 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:24:32,919 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:24:32,919 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,36973,1733797472645
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:24:32,920 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:24:32,921 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797502922
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:24:32,922 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,923 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:32,923 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:24:32,923 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:24:32,923 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:24:32,923 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:24:32,923 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:24:32,923 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:24:32,924 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797472923,5,FailOnTimeoutGroup]
2024-12-10T02:24:32,924 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797472924,5,FailOnTimeoutGroup]
2024-12-10T02:24:32,924 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,924 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,924 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:24:32,924 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,924 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,924 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:24:32,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:24:32,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:24:32,933 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:24:32,933 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b
2024-12-10T02:24:32,939 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(746): ClusterId : 7b6df85f-0950-4492-b666-ba1581539109
2024-12-10T02:24:32,939 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:24:32,941 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:24:32,941 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:24:32,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:24:32,942 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:24:32,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:32,944 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:24:32,946 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:24:32,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,946 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:32,947 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:24:32,948 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:24:32,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:32,949 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:24:32,950 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:24:32,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:32,951 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:24:32,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:24:32,951 DEBUG [RS:0;d9f49988d155:39891 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f737a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:24:32,952 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:24:32,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:32,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:32,953 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:24:32,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740
2024-12-10T02:24:32,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740
2024-12-10T02:24:32,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:24:32,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:24:32,957 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:24:32,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:24:32,961 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:24:32,962 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=800018, jitterRate=0.017276212573051453}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:24:32,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797472943Initializing all the Stores at 1733797472944 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797472944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797472944Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797472944Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797472944Cleaning up temporary data from old regions at 1733797472956 (+12 ms)Region opened successfully at 1733797472963 (+7 ms)
2024-12-10T02:24:32,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:24:32,964 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:24:32,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:24:32,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:24:32,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:24:32,964 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:32,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797472964Disabling compacts and flushes for region at 1733797472964Disabling writes for close at 1733797472964Writing region close event to WAL at 1733797472964Closed at 1733797472964
2024-12-10T02:24:32,965 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:39891
2024-12-10T02:24:32,965 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:24:32,965 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:24:32,965 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:24:32,966 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,36973,1733797472645 with port=39891, startcode=1733797472711
2024-12-10T02:24:32,966 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:32,966 DEBUG [RS:0;d9f49988d155:39891 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:24:32,966 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:24:32,967 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:24:32,969 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:24:32,970 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47827, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:24:32,970 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,39891,1733797472711
2024-12-10T02:24:32,970 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36973 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,39891,1733797472711
2024-12-10T02:24:32,971 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:24:32,972 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b
2024-12-10T02:24:32,972 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44885
2024-12-10T02:24:32,972 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:24:32,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:24:32,975 DEBUG [RS:0;d9f49988d155:39891 {}] zookeeper.ZKUtil(111): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,39891,1733797472711
2024-12-10T02:24:32,975 WARN  [RS:0;d9f49988d155:39891 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:24:32,975 INFO  [RS:0;d9f49988d155:39891 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:32,975 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/WALs/d9f49988d155,39891,1733797472711
2024-12-10T02:24:32,975 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,39891,1733797472711]
2024-12-10T02:24:32,979 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:24:32,982 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:24:32,982 INFO  [RS:0;d9f49988d155:39891 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:24:32,982 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,983 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:24:32,984 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:24:32,984 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,984 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:32,985 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:32,985 DEBUG [RS:0;d9f49988d155:39891 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:32,988 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,39891,1733797472711-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:24:33,004 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:24:33,004 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,39891,1733797472711-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,004 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,004 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.Replication(171): d9f49988d155,39891,1733797472711 started
2024-12-10T02:24:33,019 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,019 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,39891,1733797472711, RpcServer on d9f49988d155/172.17.0.2:39891, sessionid=0x1019a2fd4010001
2024-12-10T02:24:33,020 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:24:33,020 DEBUG [RS:0;d9f49988d155:39891 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,020 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,39891,1733797472711'
2024-12-10T02:24:33,020 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:24:33,020 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:24:33,021 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:24:33,021 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:24:33,021 DEBUG [RS:0;d9f49988d155:39891 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,021 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,39891,1733797472711'
2024-12-10T02:24:33,021 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:24:33,022 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:24:33,022 DEBUG [RS:0;d9f49988d155:39891 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:24:33,022 INFO  [RS:0;d9f49988d155:39891 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:24:33,022 INFO  [RS:0;d9f49988d155:39891 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:24:33,122 WARN  [d9f49988d155:36973 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:24:33,125 INFO  [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C39891%2C1733797472711, suffix=, logDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/WALs/d9f49988d155,39891,1733797472711, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/oldWALs, maxLogs=32
2024-12-10T02:24:33,127 INFO  [RS:0;d9f49988d155:39891 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C39891%2C1733797472711.1733797473127
2024-12-10T02:24:33,134 INFO  [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/WALs/d9f49988d155,39891,1733797472711/d9f49988d155%2C39891%2C1733797472711.1733797473127
2024-12-10T02:24:33,135 DEBUG [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34929:34929),(127.0.0.1/127.0.0.1:36657:36657)]
2024-12-10T02:24:33,372 DEBUG [d9f49988d155:36973 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:24:33,373 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,374 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,39891,1733797472711, state=OPENING
2024-12-10T02:24:33,376 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:24:33,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:33,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:33,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:33,378 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:24:33,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:33,379 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,39891,1733797472711}]
2024-12-10T02:24:33,531 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:24:33,534 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55347, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:24:33,538 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:24:33,538 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:33,540 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C39891%2C1733797472711.meta, suffix=.meta, logDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/WALs/d9f49988d155,39891,1733797472711, archiveDir=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/oldWALs, maxLogs=32
2024-12-10T02:24:33,542 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C39891%2C1733797472711.meta.1733797473542.meta
2024-12-10T02:24:33,548 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/WALs/d9f49988d155,39891,1733797472711/d9f49988d155%2C39891%2C1733797472711.meta.1733797473542.meta
2024-12-10T02:24:33,552 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36657:36657),(127.0.0.1/127.0.0.1:34929:34929)]
2024-12-10T02:24:33,555 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:24:33,555 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:24:33,555 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:24:33,556 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:24:33,556 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:24:33,556 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:33,556 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:24:33,556 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:24:33,558 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:24:33,559 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:24:33,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:33,560 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:33,560 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:24:33,561 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:24:33,561 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:33,561 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:33,561 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:24:33,562 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:24:33,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:33,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:33,563 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:24:33,564 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:24:33,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:33,564 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:33,564 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:24:33,565 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740
2024-12-10T02:24:33,566 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740
2024-12-10T02:24:33,568 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:24:33,568 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:24:33,569 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:24:33,570 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:24:33,571 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=858540, jitterRate=0.0916910171508789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:24:33,571 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:24:33,572 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797473556Writing region info on filesystem at 1733797473556Initializing all the Stores at 1733797473557 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797473557Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797473558 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797473558Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797473558Cleaning up temporary data from old regions at 1733797473568 (+10 ms)Running coprocessor post-open hooks at 1733797473571 (+3 ms)Region opened successfully at 1733797473572 (+1 ms)
2024-12-10T02:24:33,574 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797473531
2024-12-10T02:24:33,577 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:24:33,577 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:24:33,578 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,579 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,39891,1733797472711, state=OPEN
2024-12-10T02:24:33,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:24:33,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:24:33,584 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:33,585 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:33,588 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:24:33,588 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,39891,1733797472711 in 205 msec
2024-12-10T02:24:33,591 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:24:33,591 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 621 msec
2024-12-10T02:24:33,592 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:33,592 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:24:33,594 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:24:33,594 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,39891,1733797472711, seqNum=-1]
2024-12-10T02:24:33,594 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:24:33,596 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40297, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:24:33,602 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 683 msec
2024-12-10T02:24:33,602 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797473602, completionTime=-1
2024-12-10T02:24:33,602 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:24:33,602 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:24:33,604 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:24:33,604 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797533604
2024-12-10T02:24:33,604 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797593604
2024-12-10T02:24:33,604 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:36973, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,605 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,607 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:24:33,609 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.843sec
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:24:33,610 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:24:33,612 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:24:33,612 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:24:33,613 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36973,1733797472645-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:33,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffb657f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:33,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,36973,-1 for getting cluster id
2024-12-10T02:24:33,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:24:33,641 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7b6df85f-0950-4492-b666-ba1581539109'
2024-12-10T02:24:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:24:33,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7b6df85f-0950-4492-b666-ba1581539109"

2024-12-10T02:24:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ad383a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,36973,-1]
2024-12-10T02:24:33,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:24:33,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,645 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:24:33,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5d9cb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:33,647 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:24:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,39891,1733797472711, seqNum=-1]
2024-12-10T02:24:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:24:33,650 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:24:33,652 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,36973,1733797472645
2024-12-10T02:24:33,652 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:33,656 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:24:33,656 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:24:33,656 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:24:33,656 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:33,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,656 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:24:33,656 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:24:33,656 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1035653102, stopped=false
2024-12-10T02:24:33,657 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,36973,1733797472645
2024-12-10T02:24:33,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:33,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:33,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:33,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:33,658 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:24:33,659 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:24:33,659 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:33,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:33,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:33,659 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,39891,1733797472711' *****
2024-12-10T02:24:33,659 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:24:33,659 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:24:33,660 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:39891.
2024-12-10T02:24:33,660 DEBUG [RS:0;d9f49988d155:39891 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:24:33,660 DEBUG [RS:0;d9f49988d155:39891 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:24:33,660 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:24:33,661 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T02:24:33,661 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-10T02:24:33,661 DEBUG [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T02:24:33,661 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:24:33,661 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:24:33,661 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:24:33,661 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:24:33,661 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:24:33,661 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-10T02:24:33,680 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/.tmp/ns/1ccef4f88c7447b894da681fc2600dd8 is 43, key is default/ns:d/1733797473596/Put/seqid=0
2024-12-10T02:24:33,686 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741835_1011 (size=5153)
2024-12-10T02:24:33,686 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741835_1011 (size=5153)
2024-12-10T02:24:33,686 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/.tmp/ns/1ccef4f88c7447b894da681fc2600dd8
2024-12-10T02:24:33,693 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/.tmp/ns/1ccef4f88c7447b894da681fc2600dd8 as hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/ns/1ccef4f88c7447b894da681fc2600dd8
2024-12-10T02:24:33,700 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/ns/1ccef4f88c7447b894da681fc2600dd8, entries=2, sequenceid=6, filesize=5.0 K
2024-12-10T02:24:33,702 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false
2024-12-10T02:24:33,707 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-10T02:24:33,708 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:24:33,708 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:33,708 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797473661Running coprocessor pre-close hooks at 1733797473661Disabling compacts and flushes for region at 1733797473661Disabling writes for close at 1733797473661Obtaining lock to block concurrent updates at 1733797473661Preparing flush snapshotting stores in 1588230740 at 1733797473661Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733797473662 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733797473662Flushing 1588230740/ns: creating writer at 1733797473663 (+1 ms)Flushing 1588230740/ns: appending metadata at 1733797473679 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733797473679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b723535: reopening flushed file at 1733797473692 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false at 1733797473702 (+10 ms)Writing region close event to WAL at 1733797473703 (+1 ms)Running coprocessor post-close hooks at 1733797473708 (+5 ms)Closed at 1733797473708
2024-12-10T02:24:33,709 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:33,861 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,39891,1733797472711; all regions closed.
2024-12-10T02:24:33,862 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,862 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,862 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,862 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,862 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741834_1010 (size=1152)
2024-12-10T02:24:33,865 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741834_1010 (size=1152)
2024-12-10T02:24:33,867 DEBUG [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/oldWALs
2024-12-10T02:24:33,868 INFO  [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C39891%2C1733797472711.meta:.meta(num 1733797473542)
2024-12-10T02:24:33,868 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,868 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,868 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,868 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,868 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:33,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741833_1009 (size=93)
2024-12-10T02:24:33,871 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741833_1009 (size=93)
2024-12-10T02:24:33,873 DEBUG [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/oldWALs
2024-12-10T02:24:33,873 INFO  [RS:0;d9f49988d155:39891 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C39891%2C1733797472711:(num 1733797473127)
2024-12-10T02:24:33,873 DEBUG [RS:0;d9f49988d155:39891 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:33,873 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:24:33,873 INFO  [RS:0;d9f49988d155:39891 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:24:33,873 INFO  [RS:0;d9f49988d155:39891 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T02:24:33,874 INFO  [RS:0;d9f49988d155:39891 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:24:33,874 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:24:33,874 INFO  [RS:0;d9f49988d155:39891 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39891
2024-12-10T02:24:33,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,39891,1733797472711
2024-12-10T02:24:33,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:24:33,876 INFO  [RS:0;d9f49988d155:39891 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:24:33,878 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,39891,1733797472711]
2024-12-10T02:24:33,881 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,39891,1733797472711 already deleted, retry=false
2024-12-10T02:24:33,881 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,39891,1733797472711 expired; onlineServers=0
2024-12-10T02:24:33,881 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,36973,1733797472645' *****
2024-12-10T02:24:33,881 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:24:33,881 INFO  [M:0;d9f49988d155:36973 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:24:33,881 INFO  [M:0;d9f49988d155:36973 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:24:33,881 DEBUG [M:0;d9f49988d155:36973 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:24:33,881 DEBUG [M:0;d9f49988d155:36973 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:24:33,881 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:24:33,881 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797472924 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797472924,5,FailOnTimeoutGroup]
2024-12-10T02:24:33,881 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797472923 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797472923,5,FailOnTimeoutGroup]
2024-12-10T02:24:33,882 INFO  [M:0;d9f49988d155:36973 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:24:33,882 INFO  [M:0;d9f49988d155:36973 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:24:33,882 DEBUG [M:0;d9f49988d155:36973 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:24:33,882 INFO  [M:0;d9f49988d155:36973 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:24:33,882 INFO  [M:0;d9f49988d155:36973 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:24:33,882 INFO  [M:0;d9f49988d155:36973 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:24:33,882 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:24:33,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:24:33,884 DEBUG [M:0;d9f49988d155:36973 {}] zookeeper.ZKUtil(347): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:24:33,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:33,884 WARN  [M:0;d9f49988d155:36973 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:24:33,884 INFO  [M:0;d9f49988d155:36973 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/.lastflushedseqids
2024-12-10T02:24:33,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741836_1012 (size=99)
2024-12-10T02:24:33,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741836_1012 (size=99)
2024-12-10T02:24:33,891 INFO  [M:0;d9f49988d155:36973 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:24:33,891 INFO  [M:0;d9f49988d155:36973 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:24:33,891 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:24:33,891 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:33,891 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:33,891 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:24:33,891 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:33,891 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-10T02:24:33,910 DEBUG [M:0;d9f49988d155:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c073a3f249d44a285c328d3641cfddd is 82, key is hbase:meta,,1/info:regioninfo/1733797473578/Put/seqid=0
2024-12-10T02:24:33,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741837_1013 (size=5672)
2024-12-10T02:24:33,915 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741837_1013 (size=5672)
2024-12-10T02:24:33,916 INFO  [M:0;d9f49988d155:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c073a3f249d44a285c328d3641cfddd
2024-12-10T02:24:33,938 DEBUG [M:0;d9f49988d155:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e298f97910b44fda9c3a338f793cc02 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733797473601/Put/seqid=0
2024-12-10T02:24:33,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741838_1014 (size=5275)
2024-12-10T02:24:33,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741838_1014 (size=5275)
2024-12-10T02:24:33,945 INFO  [M:0;d9f49988d155:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e298f97910b44fda9c3a338f793cc02
2024-12-10T02:24:33,967 DEBUG [M:0;d9f49988d155:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/087d989274a54fae841614ca01d62b8e is 69, key is d9f49988d155,39891,1733797472711/rs:state/1733797472971/Put/seqid=0
2024-12-10T02:24:33,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741839_1015 (size=5156)
2024-12-10T02:24:33,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741839_1015 (size=5156)
2024-12-10T02:24:33,973 INFO  [M:0;d9f49988d155:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/087d989274a54fae841614ca01d62b8e
2024-12-10T02:24:33,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:33,978 INFO  [RS:0;d9f49988d155:39891 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:24:33,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39891-0x1019a2fd4010001, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:33,978 INFO  [RS:0;d9f49988d155:39891 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,39891,1733797472711; zookeeper connection closed.
2024-12-10T02:24:33,979 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3a45321f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3a45321f
2024-12-10T02:24:33,979 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:24:33,994 DEBUG [M:0;d9f49988d155:36973 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d732b364f96840d9acb9cfd27b4a0c5b is 52, key is load_balancer_on/state:d/1733797473654/Put/seqid=0
2024-12-10T02:24:33,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741840_1016 (size=5056)
2024-12-10T02:24:33,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741840_1016 (size=5056)
2024-12-10T02:24:33,999 INFO  [M:0;d9f49988d155:36973 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d732b364f96840d9acb9cfd27b4a0c5b
2024-12-10T02:24:34,005 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9c073a3f249d44a285c328d3641cfddd as hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c073a3f249d44a285c328d3641cfddd
2024-12-10T02:24:34,011 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9c073a3f249d44a285c328d3641cfddd, entries=8, sequenceid=29, filesize=5.5 K
2024-12-10T02:24:34,012 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7e298f97910b44fda9c3a338f793cc02 as hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e298f97910b44fda9c3a338f793cc02
2024-12-10T02:24:34,018 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7e298f97910b44fda9c3a338f793cc02, entries=3, sequenceid=29, filesize=5.2 K
2024-12-10T02:24:34,019 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/087d989274a54fae841614ca01d62b8e as hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/087d989274a54fae841614ca01d62b8e
2024-12-10T02:24:34,024 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/087d989274a54fae841614ca01d62b8e, entries=1, sequenceid=29, filesize=5.0 K
2024-12-10T02:24:34,025 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/d732b364f96840d9acb9cfd27b4a0c5b as hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d732b364f96840d9acb9cfd27b4a0c5b
2024-12-10T02:24:34,030 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44885/user/jenkins/test-data/df0247ba-65cf-0029-35a1-3def67cc611b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/d732b364f96840d9acb9cfd27b4a0c5b, entries=1, sequenceid=29, filesize=4.9 K
2024-12-10T02:24:34,031 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 140ms, sequenceid=29, compaction requested=false
2024-12-10T02:24:34,033 INFO  [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:34,033 DEBUG [M:0;d9f49988d155:36973 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797473891Disabling compacts and flushes for region at 1733797473891Disabling writes for close at 1733797473891Obtaining lock to block concurrent updates at 1733797473891Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797473891Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733797473892 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797473892Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797473893 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797473910 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797473910Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797473922 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797473938 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797473938Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797473950 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797473966 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797473966Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797473978 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797473993 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797473993Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b60c65: reopening flushed file at 1733797474004 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bdf662f: reopening flushed file at 1733797474012 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6272e7ea: reopening flushed file at 1733797474018 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@222141c5: reopening flushed file at 1733797474024 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 140ms, sequenceid=29, compaction requested=false at 1733797474031 (+7 ms)Writing region close event to WAL at 1733797474033 (+2 ms)Closed at 1733797474033
2024-12-10T02:24:34,033 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:34,033 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:34,033 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:34,034 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:34,034 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:34,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40155 is added to blk_1073741830_1006 (size=10311)
2024-12-10T02:24:34,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46783 is added to blk_1073741830_1006 (size=10311)
2024-12-10T02:24:34,037 INFO  [M:0;d9f49988d155:36973 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:24:34,037 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:24:34,037 INFO  [M:0;d9f49988d155:36973 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36973
2024-12-10T02:24:34,037 INFO  [M:0;d9f49988d155:36973 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:24:34,139 INFO  [M:0;d9f49988d155:36973 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:24:34,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:34,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36973-0x1019a2fd4010000, quorum=127.0.0.1:52464, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:24:34,143 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@824b6ae{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:34,144 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5d7e5c06{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:34,144 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:34,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@376d199b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:34,144 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b44e274{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:34,145 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:34,145 WARN  [BP-2060907613-172.17.0.2-1733797471794 heartbeating to localhost/127.0.0.1:44885 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:34,145 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:34,146 WARN  [BP-2060907613-172.17.0.2-1733797471794 heartbeating to localhost/127.0.0.1:44885 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2060907613-172.17.0.2-1733797471794 (Datanode Uuid 30fcb61b-db82-4b05-90bd-2df1c208fe62) service to localhost/127.0.0.1:44885
2024-12-10T02:24:34,146 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data3/current/BP-2060907613-172.17.0.2-1733797471794 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:34,146 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data4/current/BP-2060907613-172.17.0.2-1733797471794 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:34,147 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:34,149 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5d4bdc00{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:34,149 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@687b21ce{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:34,149 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:34,149 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@276f8783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:34,149 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4edee9ab{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:34,150 WARN  [BP-2060907613-172.17.0.2-1733797471794 heartbeating to localhost/127.0.0.1:44885 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:34,150 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:34,150 WARN  [BP-2060907613-172.17.0.2-1733797471794 heartbeating to localhost/127.0.0.1:44885 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2060907613-172.17.0.2-1733797471794 (Datanode Uuid f5dfb3f2-c576-4c4f-b605-0362e9d32cd8) service to localhost/127.0.0.1:44885
2024-12-10T02:24:34,151 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:34,151 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data1/current/BP-2060907613-172.17.0.2-1733797471794 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:34,151 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/cluster_28fec87c-c34a-58d3-93ae-3940970a46d6/data/data2/current/BP-2060907613-172.17.0.2-1733797471794 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:34,151 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:34,157 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@511dc70f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:24:34,158 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e469283{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:34,158 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:34,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45e3157d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:34,158 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3197ca45{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:34,164 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.log.dir so I do NOT create it in target/test-data/21f69832-ec76-06fe-474e-3792d57936ce
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/29d50935-2716-deae-0766-4e88be1add4f/hadoop.tmp.dir so I do NOT create it in target/test-data/21f69832-ec76-06fe-474e-3792d57936ce
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d, deleteOnExit=true
2024-12-10T02:24:34,180 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/test.cache.data in system properties and HBase conf
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:24:34,181 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:24:34,181 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:24:34,182 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:24:34,183 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:24:34,197 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:24:34,277 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:34,283 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:34,284 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:34,284 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:34,284 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:24:34,285 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:34,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:34,286 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:34,404 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cd2a640{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-39735-hadoop-hdfs-3_4_1-tests_jar-_-any-15113279236658726074/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:24:34,404 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:39735}
2024-12-10T02:24:34,405 INFO  [Time-limited test {}] server.Server(415): Started @103463ms
2024-12-10T02:24:34,424 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:24:34,493 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:34,497 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:34,498 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:34,498 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:34,498 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:24:34,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:34,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:34,618 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ab5393f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-40293-hadoop-hdfs-3_4_1-tests_jar-_-any-6023167541372038822/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:34,619 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:40293}
2024-12-10T02:24:34,619 INFO  [Time-limited test {}] server.Server(415): Started @103677ms
2024-12-10T02:24:34,621 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:34,656 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:34,659 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:34,660 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:34,660 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:34,660 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:24:34,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:34,661 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:34,762 WARN  [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data2/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:34,762 WARN  [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data1/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:34,786 WARN  [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:34,789 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xced36e4f6611f4e4 with lease ID 0x40f35a92553cf614: Processing first storage report for DS-be6d402c-06f6-4868-8fa2-34f6c2505937 from datanode DatanodeRegistration(127.0.0.1:46005, datanodeUuid=c67f1dab-e1ec-423a-84e4-79df92694a47, infoPort=37099, infoSecurePort=0, ipcPort=36479, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:34,789 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xced36e4f6611f4e4 with lease ID 0x40f35a92553cf614: from storage DS-be6d402c-06f6-4868-8fa2-34f6c2505937 node DatanodeRegistration(127.0.0.1:46005, datanodeUuid=c67f1dab-e1ec-423a-84e4-79df92694a47, infoPort=37099, infoSecurePort=0, ipcPort=36479, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:34,789 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xced36e4f6611f4e4 with lease ID 0x40f35a92553cf614: Processing first storage report for DS-42d5b2c6-8837-4176-93bf-4b3d96047320 from datanode DatanodeRegistration(127.0.0.1:46005, datanodeUuid=c67f1dab-e1ec-423a-84e4-79df92694a47, infoPort=37099, infoSecurePort=0, ipcPort=36479, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:34,789 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xced36e4f6611f4e4 with lease ID 0x40f35a92553cf614: from storage DS-42d5b2c6-8837-4176-93bf-4b3d96047320 node DatanodeRegistration(127.0.0.1:46005, datanodeUuid=c67f1dab-e1ec-423a-84e4-79df92694a47, infoPort=37099, infoSecurePort=0, ipcPort=36479, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:34,790 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21d5e4af{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-37727-hadoop-hdfs-3_4_1-tests_jar-_-any-2262499828714437828/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:34,791 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:37727}
2024-12-10T02:24:34,791 INFO  [Time-limited test {}] server.Server(415): Started @103849ms
2024-12-10T02:24:34,792 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:34,900 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data3/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:34,900 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data4/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:34,918 WARN  [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:34,920 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ebb8b95822fb08 with lease ID 0x40f35a92553cf615: Processing first storage report for DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0 from datanode DatanodeRegistration(127.0.0.1:40547, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=43595, infoSecurePort=0, ipcPort=39015, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:34,920 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ebb8b95822fb08 with lease ID 0x40f35a92553cf615: from storage DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0 node DatanodeRegistration(127.0.0.1:40547, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=43595, infoSecurePort=0, ipcPort=39015, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:34,920 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41ebb8b95822fb08 with lease ID 0x40f35a92553cf615: Processing first storage report for DS-37337454-7f71-4c30-add1-ee1eabb6911b from datanode DatanodeRegistration(127.0.0.1:40547, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=43595, infoSecurePort=0, ipcPort=39015, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:34,920 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41ebb8b95822fb08 with lease ID 0x40f35a92553cf615: from storage DS-37337454-7f71-4c30-add1-ee1eabb6911b node DatanodeRegistration(127.0.0.1:40547, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=43595, infoSecurePort=0, ipcPort=39015, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:34,988 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:24:35,020 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce
2024-12-10T02:24:35,023 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/zookeeper_0, clientPort=60905, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:24:35,024 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60905
2024-12-10T02:24:35,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,026 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:24:35,036 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:24:35,037 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e with version=8
2024-12-10T02:24:35,037 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:24:35,040 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:24:35,041 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44417
2024-12-10T02:24:35,042 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44417 connecting to ZooKeeper ensemble=127.0.0.1:60905
2024-12-10T02:24:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444170x0, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:24:35,049 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44417-0x1019a2fdd5e0000 connected
2024-12-10T02:24:35,066 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,068 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,070 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:35,070 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e, hbase.cluster.distributed=false
2024-12-10T02:24:35,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:24:35,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44417
2024-12-10T02:24:35,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44417
2024-12-10T02:24:35,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44417
2024-12-10T02:24:35,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44417
2024-12-10T02:24:35,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44417
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:24:35,090 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:24:35,091 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40537
2024-12-10T02:24:35,092 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40537 connecting to ZooKeeper ensemble=127.0.0.1:60905
2024-12-10T02:24:35,093 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,095 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405370x0, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:24:35,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40537-0x1019a2fdd5e0001 connected
2024-12-10T02:24:35,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:24:35,100 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:24:35,101 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:24:35,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:24:35,102 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:24:35,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40537
2024-12-10T02:24:35,104 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40537
2024-12-10T02:24:35,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40537
2024-12-10T02:24:35,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40537
2024-12-10T02:24:35,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40537
2024-12-10T02:24:35,118 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:44417
2024-12-10T02:24:35,119 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:35,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:35,121 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:24:35,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,123 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:24:35,124 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,44417,1733797475040 from backup master directory
2024-12-10T02:24:35,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:35,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:24:35,129 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:24:35,129 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,133 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/hbase.id] with ID: e3545cb5-096d-4ec9-9073-0f99508ea947
2024-12-10T02:24:35,134 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/.tmp/hbase.id
2024-12-10T02:24:35,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:24:35,140 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:24:35,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/.tmp/hbase.id]:[hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/hbase.id]
2024-12-10T02:24:35,154 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:35,154 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:24:35,156 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-10T02:24:35,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,164 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:24:35,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:24:35,166 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:24:35,166 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:24:35,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:35,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:24:35,174 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:24:35,175 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store
2024-12-10T02:24:35,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:24:35,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:24:35,182 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:35,182 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:24:35,182 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797475182Disabling compacts and flushes for region at 1733797475182Disabling writes for close at 1733797475182Writing region close event to WAL at 1733797475182Closed at 1733797475182
2024-12-10T02:24:35,183 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/.initializing
2024-12-10T02:24:35,183 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,186 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C44417%2C1733797475040, suffix=, logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040, archiveDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/oldWALs, maxLogs=10
2024-12-10T02:24:35,187 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C44417%2C1733797475040.1733797475187
2024-12-10T02:24:35,192 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187
2024-12-10T02:24:35,197 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43595:43595),(127.0.0.1/127.0.0.1:37099:37099)]
2024-12-10T02:24:35,197 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:24:35,198 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:35,198 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,198 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,200 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:35,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:24:35,201 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,202 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,202 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:35,203 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:24:35,203 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,204 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:35,204 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:35,205 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:24:35,205 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,205 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:35,206 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:24:35,207 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:24:35,207 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,207 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:35,207 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,208 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,209 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,211 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,211 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:24:35,213 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:24:35,216 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:24:35,217 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760006, jitterRate=-0.03360241651535034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:24:35,218 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797475198Initializing all the Stores at 1733797475199 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475199Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797475199Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797475199Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797475199Cleaning up temporary data from old regions at 1733797475211 (+12 ms)Region opened successfully at 1733797475218 (+7 ms)
2024-12-10T02:24:35,219 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:24:35,223 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b953aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:24:35,224 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:24:35,224 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:24:35,224 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:24:35,224 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:24:35,225 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:24:35,225 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:24:35,225 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:24:35,228 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:24:35,228 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:24:35,230 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:24:35,230 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:24:35,231 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:24:35,232 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:24:35,232 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:24:35,233 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:24:35,236 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:24:35,237 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:24:35,239 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:24:35,241 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:24:35,242 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:24:35,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:35,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:24:35,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,244 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,44417,1733797475040, sessionid=0x1019a2fdd5e0000, setting cluster-up flag (Was=false)
2024-12-10T02:24:35,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,254 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:24:35,255 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,264 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:24:35,265 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,44417,1733797475040
2024-12-10T02:24:35,267 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:24:35,268 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:35,269 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:24:35,269 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:24:35,269 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,44417,1733797475040
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:24:35,270 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:35,270 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:35,270 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:35,270 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:24:35,271 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:24:35,271 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,271 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:24:35,271 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:35,273 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797505273
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:24:35,273 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,274 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:24:35,274 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,274 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:24:35,274 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:24:35,274 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:24:35,274 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:24:35,274 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:24:35,275 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797475274,5,FailOnTimeoutGroup]
2024-12-10T02:24:35,276 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797475275,5,FailOnTimeoutGroup]
2024-12-10T02:24:35,276 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,276 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:24:35,276 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,276 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:24:35,284 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:24:35,287 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:24:35,288 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e
2024-12-10T02:24:35,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:24:35,296 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:24:35,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:35,298 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:24:35,299 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:24:35,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,300 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,300 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:24:35,301 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:24:35,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,302 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,302 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:24:35,304 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:24:35,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,304 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,304 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:24:35,306 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:24:35,306 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,306 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:24:35,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740
2024-12-10T02:24:35,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740
2024-12-10T02:24:35,308 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(746): ClusterId : e3545cb5-096d-4ec9-9073-0f99508ea947
2024-12-10T02:24:35,308 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:24:35,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:24:35,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:24:35,310 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:24:35,311 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:24:35,311 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:24:35,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:24:35,313 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:24:35,314 DEBUG [RS:0;d9f49988d155:40537 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e879f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:24:35,314 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:24:35,315 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767946, jitterRate=-0.023507267236709595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797475296Initializing all the Stores at 1733797475297 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475297Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475298 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797475298Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475298Cleaning up temporary data from old regions at 1733797475309 (+11 ms)Region opened successfully at 1733797475316 (+7 ms)
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:24:35,317 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:24:35,317 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:24:35,317 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797475317Disabling compacts and flushes for region at 1733797475317Disabling writes for close at 1733797475317Writing region close event to WAL at 1733797475317Closed at 1733797475317
2024-12-10T02:24:35,319 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:35,319 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:24:35,320 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:24:35,321 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:24:35,323 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:24:35,334 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:40537
2024-12-10T02:24:35,334 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:24:35,334 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:24:35,334 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:24:35,335 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,44417,1733797475040 with port=40537, startcode=1733797475089
2024-12-10T02:24:35,335 DEBUG [RS:0;d9f49988d155:40537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:24:35,338 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:24:35,338 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44417 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,338 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44417 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,341 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e
2024-12-10T02:24:35,341 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42523
2024-12-10T02:24:35,341 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:24:35,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:24:35,345 DEBUG [RS:0;d9f49988d155:40537 {}] zookeeper.ZKUtil(111): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,345 WARN  [RS:0;d9f49988d155:40537 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:24:35,345 INFO  [RS:0;d9f49988d155:40537 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:35,345 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,345 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,40537,1733797475089]
2024-12-10T02:24:35,351 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:24:35,353 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:24:35,355 INFO  [RS:0;d9f49988d155:40537 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:24:35,355 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,355 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:24:35,356 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:24:35,356 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,356 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,356 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:35,357 DEBUG [RS:0;d9f49988d155:40537 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:35,359 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,359 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,359 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,359 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,359 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,360 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,40537,1733797475089-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:24:35,383 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:24:35,384 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,40537,1733797475089-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,384 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,384 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.Replication(171): d9f49988d155,40537,1733797475089 started
2024-12-10T02:24:35,400 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,401 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,40537,1733797475089, RpcServer on d9f49988d155/172.17.0.2:40537, sessionid=0x1019a2fdd5e0001
2024-12-10T02:24:35,401 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:24:35,401 DEBUG [RS:0;d9f49988d155:40537 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,401 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,40537,1733797475089'
2024-12-10T02:24:35,401 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,40537,1733797475089'
2024-12-10T02:24:35,402 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:24:35,403 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:24:35,403 DEBUG [RS:0;d9f49988d155:40537 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:24:35,403 INFO  [RS:0;d9f49988d155:40537 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:24:35,403 INFO  [RS:0;d9f49988d155:40537 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:24:35,473 WARN  [d9f49988d155:44417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:24:35,506 INFO  [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C40537%2C1733797475089, suffix=, logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089, archiveDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs, maxLogs=32
2024-12-10T02:24:35,507 INFO  [RS:0;d9f49988d155:40537 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797475506
2024-12-10T02:24:35,514 INFO  [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506
2024-12-10T02:24:35,518 DEBUG [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:43595:43595)]
2024-12-10T02:24:35,723 DEBUG [d9f49988d155:44417 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:24:35,724 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,725 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,40537,1733797475089, state=OPENING
2024-12-10T02:24:35,727 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:24:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:24:35,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:24:35,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:35,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:35,729 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,40537,1733797475089}]
2024-12-10T02:24:35,883 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:24:35,885 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:24:35,889 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:24:35,889 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:35,891 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C40537%2C1733797475089.meta, suffix=.meta, logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089, archiveDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs, maxLogs=32
2024-12-10T02:24:35,892 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
2024-12-10T02:24:35,897 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
2024-12-10T02:24:35,900 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:43595:43595)]
2024-12-10T02:24:35,901 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:24:35,901 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:24:35,901 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:24:35,901 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:24:35,901 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:24:35,901 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:35,902 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:24:35,902 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:24:35,903 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:24:35,904 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:24:35,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,905 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,905 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:24:35,906 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:24:35,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,906 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,906 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:24:35,907 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:24:35,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,907 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,908 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:24:35,908 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:24:35,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:35,909 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:24:35,909 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:24:35,910 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740
2024-12-10T02:24:35,911 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740
2024-12-10T02:24:35,913 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:24:35,913 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:24:35,913 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:24:35,915 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:24:35,916 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=733352, jitterRate=-0.06749571859836578}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:24:35,916 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:24:35,917 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797475902Writing region info on filesystem at 1733797475902Initializing all the Stores at 1733797475903 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475903Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475903Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797475903Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797475903Cleaning up temporary data from old regions at 1733797475913 (+10 ms)Running coprocessor post-open hooks at 1733797475916 (+3 ms)Region opened successfully at 1733797475917 (+1 ms)
2024-12-10T02:24:35,945 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797475882
2024-12-10T02:24:35,947 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:24:35,947 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:24:35,948 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,949 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,40537,1733797475089, state=OPEN
2024-12-10T02:24:35,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:24:35,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:24:35,956 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,40537,1733797475089
2024-12-10T02:24:35,956 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:35,956 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:24:35,958 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:24:35,958 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,40537,1733797475089 in 227 msec
2024-12-10T02:24:35,961 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:24:35,961 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 638 msec
2024-12-10T02:24:35,962 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:24:35,962 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:24:35,963 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:24:35,964 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,40537,1733797475089, seqNum=-1]
2024-12-10T02:24:35,964 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:24:35,965 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:24:35,970 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 702 msec
2024-12-10T02:24:35,971 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797475971, completionTime=-1
2024-12-10T02:24:35,971 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:24:35,971 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:24:35,972 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:24:35,972 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797535972
2024-12-10T02:24:35,972 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797595972
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:44417, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,973 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:35,975 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:24:35,976 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.847sec
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:24:35,977 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:24:35,979 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:24:35,979 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:24:35,979 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44417,1733797475040-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd7b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:36,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,44417,-1 for getting cluster id
2024-12-10T02:24:36,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:24:36,010 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e3545cb5-096d-4ec9-9073-0f99508ea947'
2024-12-10T02:24:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:24:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e3545cb5-096d-4ec9-9073-0f99508ea947"

2024-12-10T02:24:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1535ec62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,44417,-1]
2024-12-10T02:24:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:24:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:24:36,013 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:24:36,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abbe752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:24:36,014 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:24:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,40537,1733797475089, seqNum=-1]
2024-12-10T02:24:36,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:24:36,017 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:24:36,019 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,44417,1733797475040
2024-12-10T02:24:36,019 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:36,021 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:24:36,037 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:24:36,038 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:24:36,038 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:24:36,038 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33309
2024-12-10T02:24:36,039 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33309 connecting to ZooKeeper ensemble=127.0.0.1:60905
2024-12-10T02:24:36,040 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:36,042 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:24:36,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333090x0, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:24:36,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:333090x0, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-10T02:24:36,046 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-10T02:24:36,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33309-0x1019a2fdd5e0002 connected
2024-12-10T02:24:36,047 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:24:36,048 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:24:36,048 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:24:36,049 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:24:36,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33309
2024-12-10T02:24:36,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33309
2024-12-10T02:24:36,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33309
2024-12-10T02:24:36,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33309
2024-12-10T02:24:36,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33309
2024-12-10T02:24:36,056 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(746): ClusterId : e3545cb5-096d-4ec9-9073-0f99508ea947
2024-12-10T02:24:36,056 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:24:36,058 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:24:36,058 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:24:36,060 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:24:36,060 DEBUG [RS:1;d9f49988d155:33309 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7257e1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:24:36,074 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d9f49988d155:33309
2024-12-10T02:24:36,075 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:24:36,075 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:24:36,075 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:24:36,075 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,44417,1733797475040 with port=33309, startcode=1733797476037
2024-12-10T02:24:36,075 DEBUG [RS:1;d9f49988d155:33309 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:24:36,077 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55593, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:24:36,077 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44417 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,077 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44417 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,079 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e
2024-12-10T02:24:36,079 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42523
2024-12-10T02:24:36,079 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:24:36,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:24:36,082 DEBUG [RS:1;d9f49988d155:33309 {}] zookeeper.ZKUtil(111): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,082 WARN  [RS:1;d9f49988d155:33309 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:24:36,082 INFO  [RS:1;d9f49988d155:33309 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:24:36,082 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,33309,1733797476037]
2024-12-10T02:24:36,083 DEBUG [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,086 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:24:36,088 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:24:36,088 INFO  [RS:1;d9f49988d155:33309 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:24:36,088 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,089 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:24:36,089 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:24:36,089 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,089 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:36,090 DEBUG [RS:1;d9f49988d155:33309 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:24:36,090 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,090 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,091 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,091 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,091 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,091 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,33309,1733797476037-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:24:36,105 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:24:36,105 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,33309,1733797476037-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,106 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,106 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.Replication(171): d9f49988d155,33309,1733797476037 started
2024-12-10T02:24:36,119 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:24:36,119 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,33309,1733797476037, RpcServer on d9f49988d155/172.17.0.2:33309, sessionid=0x1019a2fdd5e0002
2024-12-10T02:24:36,119 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:24:36,119 DEBUG [RS:1;d9f49988d155:33309 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,119 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,33309,1733797476037'
2024-12-10T02:24:36,119 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;d9f49988d155:33309,5,FailOnTimeoutGroup]
2024-12-10T02:24:36,119 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:24:36,120 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:24:36,120 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,33309,1733797476037
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,33309,1733797476037'
2024-12-10T02:24:36,120 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:24:36,121 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:24:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is d9f49988d155,44417,1733797475040
2024-12-10T02:24:36,122 DEBUG [RS:1;d9f49988d155:33309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:24:36,122 INFO  [RS:1;d9f49988d155:33309 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:24:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fa8e196
2024-12-10T02:24:36,122 INFO  [RS:1;d9f49988d155:33309 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:24:36,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T02:24:36,123 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T02:24:36,124 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T02:24:36,124 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T02:24:36,124 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:24:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T02:24:36,127 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T02:24:36,127 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:36,127 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-12-10T02:24:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:24:36,129 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T02:24:36,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741835_1011 (size=393)
2024-12-10T02:24:36,135 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741835_1011 (size=393)
2024-12-10T02:24:36,137 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18839f6a2f527eb0eaba0d7ea45ab82a, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e
2024-12-10T02:24:36,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40547 is added to blk_1073741836_1012 (size=76)
2024-12-10T02:24:36,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46005 is added to blk_1073741836_1012 (size=76)
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 18839f6a2f527eb0eaba0d7ea45ab82a, disabling compactions & flushes
2024-12-10T02:24:36,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a. after waiting 0 ms
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,144 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,144 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18839f6a2f527eb0eaba0d7ea45ab82a:
Waiting for close lock at 1733797476144Disabling compacts and flushes for region at 1733797476144Disabling writes for close at 1733797476144Writing region close event to WAL at 1733797476144Closed at 1733797476144
2024-12-10T02:24:36,146 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T02:24:36,146 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1733797476146"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797476146"}]},"ts":"1733797476146"}
2024-12-10T02:24:36,149 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T02:24:36,150 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T02:24:36,150 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797476150"}]},"ts":"1733797476150"}
2024-12-10T02:24:36,152 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-12-10T02:24:36,152 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=18839f6a2f527eb0eaba0d7ea45ab82a, ASSIGN}]
2024-12-10T02:24:36,154 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=18839f6a2f527eb0eaba0d7ea45ab82a, ASSIGN
2024-12-10T02:24:36,155 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=18839f6a2f527eb0eaba0d7ea45ab82a, ASSIGN; state=OFFLINE, location=d9f49988d155,40537,1733797475089; forceNewPlan=false, retain=false
2024-12-10T02:24:36,223 INFO  [RS:1;d9f49988d155:33309 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C33309%2C1733797476037, suffix=, logDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037, archiveDir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs, maxLogs=32
2024-12-10T02:24:36,224 INFO  [RS:1;d9f49988d155:33309 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C33309%2C1733797476037.1733797476224
2024-12-10T02:24:36,229 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,230 INFO  [RS:1;d9f49988d155:33309 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
2024-12-10T02:24:36,231 DEBUG [RS:1;d9f49988d155:33309 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37099:37099),(127.0.0.1/127.0.0.1:43595:43595)]
2024-12-10T02:24:36,233 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,306 INFO  [d9f49988d155:44417 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-10T02:24:36,306 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18839f6a2f527eb0eaba0d7ea45ab82a, regionState=OPENING, regionLocation=d9f49988d155,40537,1733797475089
2024-12-10T02:24:36,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=18839f6a2f527eb0eaba0d7ea45ab82a, ASSIGN because future has completed
2024-12-10T02:24:36,310 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18839f6a2f527eb0eaba0d7ea45ab82a, server=d9f49988d155,40537,1733797475089}]
2024-12-10T02:24:36,467 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,467 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 18839f6a2f527eb0eaba0d7ea45ab82a, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:24:36,467 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,468 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:24:36,468 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,468 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,469 INFO  [StoreOpener-18839f6a2f527eb0eaba0d7ea45ab82a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 18839f6a2f527eb0eaba0d7ea45ab82a 
2024-12-10T02:24:36,471 INFO  [StoreOpener-18839f6a2f527eb0eaba0d7ea45ab82a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18839f6a2f527eb0eaba0d7ea45ab82a columnFamilyName info
2024-12-10T02:24:36,471 DEBUG [StoreOpener-18839f6a2f527eb0eaba0d7ea45ab82a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:24:36,471 INFO  [StoreOpener-18839f6a2f527eb0eaba0d7ea45ab82a-1 {}] regionserver.HStore(327): Store=18839f6a2f527eb0eaba0d7ea45ab82a/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:24:36,471 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,472 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,472 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,473 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,473 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,474 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,477 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:24:36,477 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 18839f6a2f527eb0eaba0d7ea45ab82a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=787975, jitterRate=0.0019622594118118286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:24:36,477 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:36,478 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 18839f6a2f527eb0eaba0d7ea45ab82a:
Running coprocessor pre-open hook at 1733797476468Writing region info on filesystem at 1733797476468Initializing all the Stores at 1733797476469 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797476469Cleaning up temporary data from old regions at 1733797476473 (+4 ms)Running coprocessor post-open hooks at 1733797476477 (+4 ms)Region opened successfully at 1733797476478 (+1 ms)
2024-12-10T02:24:36,479 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., pid=6, masterSystemTime=1733797476462
2024-12-10T02:24:36,481 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,481 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:36,482 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18839f6a2f527eb0eaba0d7ea45ab82a, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,40537,1733797475089
2024-12-10T02:24:36,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18839f6a2f527eb0eaba0d7ea45ab82a, server=d9f49988d155,40537,1733797475089 because future has completed
2024-12-10T02:24:36,488 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T02:24:36,488 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 18839f6a2f527eb0eaba0d7ea45ab82a, server=d9f49988d155,40537,1733797475089 in 177 msec
2024-12-10T02:24:36,491 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T02:24:36,491 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=18839f6a2f527eb0eaba0d7ea45ab82a, ASSIGN in 336 msec
2024-12-10T02:24:36,492 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T02:24:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797476492"}]},"ts":"1733797476492"}
2024-12-10T02:24:36,494 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-12-10T02:24:36,495 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T02:24:36,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 371 msec
2024-12-10T02:24:36,743 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:24:36,745 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,765 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:36,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T02:24:36,773 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-12-10T02:24:36,773 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-12-10T02:24:41,351 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:24:41,351 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-12-10T02:24:41,969 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:24:41,971 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:41,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:41,993 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:41,994 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:24:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:24:46,164 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-12-10T02:24:46,164 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-12-10T02:24:46,168 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T02:24:46,168 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:46,181 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:46,184 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:46,185 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:46,185 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:46,185 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:46,185 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36d0b5ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:46,186 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@51f59516{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:46,303 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c5438f9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-33785-hadoop-hdfs-3_4_1-tests_jar-_-any-17938651862729661225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:46,303 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e7a8425{HTTP/1.1, (http/1.1)}{localhost:33785}
2024-12-10T02:24:46,303 INFO  [Time-limited test {}] server.Server(415): Started @115361ms
2024-12-10T02:24:46,305 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:46,333 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:46,336 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:46,337 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:46,337 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:46,337 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:46,338 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@712f5f14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:46,338 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3afd309b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:46,400 WARN  [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data5/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,400 WARN  [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data6/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,418 WARN  [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd46321cbe158becd with lease ID 0x40f35a92553cf616: Processing first storage report for DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91 from datanode DatanodeRegistration(127.0.0.1:40783, datanodeUuid=abb4a91d-473f-4ae3-a11c-78b87947509c, infoPort=45895, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd46321cbe158becd with lease ID 0x40f35a92553cf616: from storage DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91 node DatanodeRegistration(127.0.0.1:40783, datanodeUuid=abb4a91d-473f-4ae3-a11c-78b87947509c, infoPort=45895, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd46321cbe158becd with lease ID 0x40f35a92553cf616: Processing first storage report for DS-ef5984e4-2a0f-4fc1-b8b4-f2308dd3abe3 from datanode DatanodeRegistration(127.0.0.1:40783, datanodeUuid=abb4a91d-473f-4ae3-a11c-78b87947509c, infoPort=45895, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd46321cbe158becd with lease ID 0x40f35a92553cf616: from storage DS-ef5984e4-2a0f-4fc1-b8b4-f2308dd3abe3 node DatanodeRegistration(127.0.0.1:40783, datanodeUuid=abb4a91d-473f-4ae3-a11c-78b87947509c, infoPort=45895, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,456 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@39835cdd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-37169-hadoop-hdfs-3_4_1-tests_jar-_-any-10091028661406925773/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:46,457 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@12b19204{HTTP/1.1, (http/1.1)}{localhost:37169}
2024-12-10T02:24:46,457 INFO  [Time-limited test {}] server.Server(415): Started @115515ms
2024-12-10T02:24:46,459 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:46,495 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:46,498 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:46,499 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:46,499 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:46,499 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:24:46,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68004957{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:46,499 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78e445ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:46,555 WARN  [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,555 WARN  [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,579 WARN  [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:46,581 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca96872aebb8bc6f with lease ID 0x40f35a92553cf617: Processing first storage report for DS-78ed481c-1d9e-48b8-8798-1e4695865a7f from datanode DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,581 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca96872aebb8bc6f with lease ID 0x40f35a92553cf617: from storage DS-78ed481c-1d9e-48b8-8798-1e4695865a7f node DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,581 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xca96872aebb8bc6f with lease ID 0x40f35a92553cf617: Processing first storage report for DS-55e1c650-c036-4d0d-bbb1-8af211aabe00 from datanode DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,581 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xca96872aebb8bc6f with lease ID 0x40f35a92553cf617: from storage DS-55e1c650-c036-4d0d-bbb1-8af211aabe00 node DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,617 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea37f0d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-45525-hadoop-hdfs-3_4_1-tests_jar-_-any-4193249741745236385/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:46,618 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15010086{HTTP/1.1, (http/1.1)}{localhost:45525}
2024-12-10T02:24:46,618 INFO  [Time-limited test {}] server.Server(415): Started @115676ms
2024-12-10T02:24:46,619 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:46,713 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data9/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,713 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data10/current/BP-35725305-172.17.0.2-1733797474215/current, will proceed with Du for space computation calculation, 
2024-12-10T02:24:46,729 WARN  [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:46,732 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99b8f0a17af028fc with lease ID 0x40f35a92553cf618: Processing first storage report for DS-96dda52a-6590-41f0-8054-8f67479a0831 from datanode DatanodeRegistration(127.0.0.1:38163, datanodeUuid=b183abda-9306-41ac-a257-30b87175491a, infoPort=43729, infoSecurePort=0, ipcPort=35609, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,732 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99b8f0a17af028fc with lease ID 0x40f35a92553cf618: from storage DS-96dda52a-6590-41f0-8054-8f67479a0831 node DatanodeRegistration(127.0.0.1:38163, datanodeUuid=b183abda-9306-41ac-a257-30b87175491a, infoPort=43729, infoSecurePort=0, ipcPort=35609, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,732 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x99b8f0a17af028fc with lease ID 0x40f35a92553cf618: Processing first storage report for DS-b3a940c1-728e-42ac-b631-ab036a0d5e18 from datanode DatanodeRegistration(127.0.0.1:38163, datanodeUuid=b183abda-9306-41ac-a257-30b87175491a, infoPort=43729, infoSecurePort=0, ipcPort=35609, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215)
2024-12-10T02:24:46,732 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x99b8f0a17af028fc with lease ID 0x40f35a92553cf618: from storage DS-b3a940c1-728e-42ac-b631-ab036a0d5e18 node DatanodeRegistration(127.0.0.1:38163, datanodeUuid=b183abda-9306-41ac-a257-30b87175491a, infoPort=43729, infoSecurePort=0, ipcPort=35609, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:46,738 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,738 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,738 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,739 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 block BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:46,739 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta block BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:46,739 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 block BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:46,739 WARN  [PacketResponder: BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40547] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,739 WARN  [PacketResponder: BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40547] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,739 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,740 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 block BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:46,740 WARN  [PacketResponder: BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40547] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:60094 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46005:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60094 dst: /127.0.0.1:46005
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1848451045_22 at /127.0.0.1:52234 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52234 dst: /127.0.0.1:40547
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1848451045_22 at /127.0.0.1:60058 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46005:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60058 dst: /127.0.0.1:46005
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:60084 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46005:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60084 dst: /127.0.0.1:46005
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2059368875_22 at /127.0.0.1:60128 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:46005:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60128 dst: /127.0.0.1:46005
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,741 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2059368875_22 at /127.0.0.1:52304 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:40547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52304 dst: /127.0.0.1:40547
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,741 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:52270 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52270 dst: /127.0.0.1:40547
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,741 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:52276 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40547:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:52276 dst: /127.0.0.1:40547
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,742 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21d5e4af{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:46,743 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@145c0180{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:46,743 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:46,743 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7720beab{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:46,743 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69bbaec1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:46,744 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:46,744 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid 2dc758ca-efcb-43af-ab9b-dd6e6afe3034) service to localhost/127.0.0.1:42523
2024-12-10T02:24:46,744 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:46,744 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:46,745 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data3/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:46,745 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data4/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:46,745 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:46,746 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta block BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,746 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 block BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,746 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 block BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,746 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@5c5e8a5 {}] datanode.DataXceiver(331): 127.0.0.1:46005:DataXceiver error processing unknown operation  src: /127.0.0.1:41778 dst: /127.0.0.1:46005
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:46,747 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 block BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,749 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ab5393f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:46,750 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47f82e76{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:46,750 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:46,750 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30e7c448{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:46,750 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@257cf4bb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:46,751 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:46,751 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid c67f1dab-e1ec-423a-84e4-79df92694a47) service to localhost/127.0.0.1:42523
2024-12-10T02:24:46,751 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:46,751 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:46,752 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data1/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:46,752 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data2/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:46,752 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., hostname=d9f49988d155,40537,1733797475089, seqNum=2]
2024-12-10T02:24:46,757 ERROR [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e-prefix:d9f49988d155,40537,1733797475089 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,757 WARN  [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e-prefix:d9f49988d155,40537,1733797475089 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,758 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,758 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C40537%2C1733797475089:(num 1733797475506) roll requested
2024-12-10T02:24:46,758 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797486758
2024-12-10T02:24:46,760 WARN  [Thread-902 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,761 WARN  [Thread-902 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:46,761 WARN  [Thread-902 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741838_1018
2024-12-10T02:24:46,763 WARN  [Thread-902 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:46,769 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:46,769 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:46,769 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:46,770 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:46,770 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:46,770 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758
2024-12-10T02:24:46,770 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,770 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:46,771 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45895:45895),(127.0.0.1/127.0.0.1:43729:43729)]
2024-12-10T02:24:46,771 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:46,771 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-12-10T02:24:46,771 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-12-10T02:24:46,772 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506
2024-12-10T02:24:46,774 WARN  [IPC Server handler 2 on default port 42523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009
2024-12-10T02:24:46,778 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 after 4ms
2024-12-10T02:24:47,168 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:48,091 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:48,771 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:48,772 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758
2024-12-10T02:24:48,773 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:48,773 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758 block BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:48,774 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:41454 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:40783:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41454 dst: /127.0.0.1:40783
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:48,774 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:55312 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:38163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55312 dst: /127.0.0.1:38163
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:48,775 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c5438f9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:48,775 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e7a8425{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:48,775 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:48,776 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@51f59516{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:48,776 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36d0b5ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:48,777 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:48,777 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:48,777 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid abb4a91d-473f-4ae3-a11c-78b87947509c) service to localhost/127.0.0.1:42523
2024-12-10T02:24:48,777 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:48,778 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data5/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:48,778 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data6/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:48,778 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:49,168 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:50,091 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:50,772 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]
2024-12-10T02:24:50,772 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:50,772 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C40537%2C1733797475089:(num 1733797486758) roll requested
2024-12-10T02:24:50,772 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797490772
2024-12-10T02:24:50,775 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:50,775 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:50,775 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741840_1022
2024-12-10T02:24:50,776 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:50,778 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 after 4006ms
2024-12-10T02:24:50,778 WARN  [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46005
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:50,778 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:36322 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data10]'}, localName='127.0.0.1:38163', datanodeUuid='b183abda-9306-41ac-a257-30b87175491a', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023 to mirror 127.0.0.1:46005
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:50,779 WARN  [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:50,779 WARN  [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023
2024-12-10T02:24:50,779 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:36322 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T02:24:50,779 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:36322 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:38163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36322 dst: /127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:50,779 WARN  [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:50,783 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-12-10T02:24:50,783 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:50,783 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:50,783 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:50,783 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:50,783 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:50,783 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797490772
2024-12-10T02:24:50,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38163 is added to blk_1073741839_1021 (size=2431)
2024-12-10T02:24:50,786 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43729:43729),(127.0.0.1/127.0.0.1:44143:44143)]
2024-12-10T02:24:50,786 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:50,786 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758 is not closed yet, will try archiving it next time
2024-12-10T02:24:51,168 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:51,186 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:52,092 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,744 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@f81e1c4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38163, datanodeUuid=b183abda-9306-41ac-a257-30b87175491a, infoPort=43729, infoSecurePort=0, ipcPort=35609, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741839_1021 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,786 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,788 WARN  [ResponseProcessor for block BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,788 WARN  [DataStreamer for file /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797490772 block BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:52,788 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:36336 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:38163:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36336 dst: /127.0.0.1:38163
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,789 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35316 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35316 dst: /127.0.0.1:44705
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea37f0d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:52,790 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15010086{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:24:52,790 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:24:52,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78e445ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:24:52,790 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68004957{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:24:52,792 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:24:52,792 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:24:52,792 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid b183abda-9306-41ac-a257-30b87175491a) service to localhost/127.0.0.1:42523
2024-12-10T02:24:52,792 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:24:52,793 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data9/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:52,793 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data10/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:24:52,793 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:24:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:52,801 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:24:52,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/fc3a2ef7a5d94e558f7a8b8034c92c05 is 1080, key is row0002/info:/1733797488779/Put/seqid=0
2024-12-10T02:24:52,822 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,822 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:52,822 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741843_1026
2024-12-10T02:24:52,823 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:52,825 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35338 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,825 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:52,825 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027
2024-12-10T02:24:52,825 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35338 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027] {}] datanode.BlockReceiver(316): Block 1073741844 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:52,825 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35338 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741844_1027] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35338 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,826 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:52,827 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,827 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:52,827 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741845_1028
2024-12-10T02:24:52,828 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:52,830 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:52,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35348 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,830 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:52,830 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029
2024-12-10T02:24:52,830 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35348 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:52,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35348 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35348 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:52,830 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:52,831 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:52,831 WARN  [IPC Server handler 3 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:52,831 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:52,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741847_1030 (size=10347)
2024-12-10T02:24:53,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:53,235 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/fc3a2ef7a5d94e558f7a8b8034c92c05
2024-12-10T02:24:53,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/fc3a2ef7a5d94e558f7a8b8034c92c05 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05
2024-12-10T02:24:53,248 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05, entries=5, sequenceid=11, filesize=10.1 K
2024-12-10T02:24:53,250 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 18839f6a2f527eb0eaba0d7ea45ab82a in 449ms, sequenceid=11, compaction requested=false
2024-12-10T02:24:53,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:53,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:53,424 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-12-10T02:24:53,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8955215aab74460cb983a2388e307c69 is 1080, key is row0007/info:/1733797492802/Put/seqid=0
2024-12-10T02:24:53,431 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:53,431 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:53,431 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741848_1031
2024-12-10T02:24:53,432 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:53,433 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:53,433 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:53,433 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741849_1032
2024-12-10T02:24:53,433 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:53,435 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:53,435 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35376 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:53,435 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:53,435 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033
2024-12-10T02:24:53,435 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35376 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:53,436 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35376 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35376 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:53,436 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:53,437 WARN  [Thread-930 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:53,437 WARN  [Thread-930 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:53,437 WARN  [Thread-930 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741851_1034
2024-12-10T02:24:53,437 WARN  [Thread-930 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:53,438 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:53,438 WARN  [IPC Server handler 3 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:53,438 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:53,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741852_1035 (size=12506)
2024-12-10T02:24:53,842 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8955215aab74460cb983a2388e307c69
2024-12-10T02:24:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8955215aab74460cb983a2388e307c69 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69
2024-12-10T02:24:53,854 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69, entries=7, sequenceid=24, filesize=12.2 K
2024-12-10T02:24:53,855 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 18839f6a2f527eb0eaba0d7ea45ab82a in 432ms, sequenceid=24, compaction requested=false
2024-12-10T02:24:53,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:53,855 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-12-10T02:24:53,855 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:53,855 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69 because midkey is the same as first or last row
2024-12-10T02:24:54,092 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,786 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]
2024-12-10T02:24:54,787 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,787 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C40537%2C1733797475089:(num 1733797490772) roll requested
2024-12-10T02:24:54,787 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797494787
2024-12-10T02:24:54,790 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,790 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:54,790 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741853_1036
2024-12-10T02:24:54,791 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:54,792 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,792 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:54,792 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741854_1037
2024-12-10T02:24:54,793 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:54,794 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,794 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:54,794 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741855_1038
2024-12-10T02:24:54,794 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:54,796 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,796 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35396 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039 to mirror 127.0.0.1:40547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:54,796 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:54,796 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039
2024-12-10T02:24:54,796 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35396 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T02:24:54,796 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35396 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35396 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:54,797 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:54,797 WARN  [IPC Server handler 2 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:54,797 WARN  [IPC Server handler 2 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:54,797 WARN  [IPC Server handler 2 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:54,799 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:54,799 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:54,799 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:54,800 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:54,800 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:54,800 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797490772 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797494787
2024-12-10T02:24:54,801 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44143:44143)]
2024-12-10T02:24:54,801 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:54,801 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797490772 is not closed yet, will try archiving it next time
2024-12-10T02:24:54,801 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797486758 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs/d9f49988d155%2C40537%2C1733797475089.1733797486758
2024-12-10T02:24:54,802 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741842_1025 (size=25992)
2024-12-10T02:24:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:54,842 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T02:24:54,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/1de715bdef8e4d61b8afdc6fa5e24cbd is 1079, key is tmprow/info:/1733797494841/Put/seqid=0
2024-12-10T02:24:54,847 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,848 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:54,848 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741858_1041
2024-12-10T02:24:54,848 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:54,849 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,849 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:54,849 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741859_1042
2024-12-10T02:24:54,850 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:54,851 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,851 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:54,851 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741860_1043
2024-12-10T02:24:54,851 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:54,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:54,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:54,852 WARN  [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741861_1044
2024-12-10T02:24:54,853 WARN  [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:54,853 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:54,853 WARN  [IPC Server handler 3 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:54,853 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:54,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741862_1045 (size=6027)
2024-12-10T02:24:55,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:55,202 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:55,257 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/1de715bdef8e4d61b8afdc6fa5e24cbd
2024-12-10T02:24:55,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/1de715bdef8e4d61b8afdc6fa5e24cbd as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd
2024-12-10T02:24:55,269 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd, entries=1, sequenceid=34, filesize=5.9 K
2024-12-10T02:24:55,270 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 18839f6a2f527eb0eaba0d7ea45ab82a in 428ms, sequenceid=34, compaction requested=true
2024-12-10T02:24:55,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:55,270 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-12-10T02:24:55,270 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:55,270 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69 because midkey is the same as first or last row
2024-12-10T02:24:55,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18839f6a2f527eb0eaba0d7ea45ab82a:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:24:55,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:55,271 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:24:55,272 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:24:55,272 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1541): 18839f6a2f527eb0eaba0d7ea45ab82a/info is initiating minor compaction (all files)
2024-12-10T02:24:55,272 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18839f6a2f527eb0eaba0d7ea45ab82a/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:55,272 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd] into tmpdir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp, totalSize=28.2 K
2024-12-10T02:24:55,273 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc3a2ef7a5d94e558f7a8b8034c92c05, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733797488779
2024-12-10T02:24:55,273 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8955215aab74460cb983a2388e307c69, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1733797492802
2024-12-10T02:24:55,273 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1de715bdef8e4d61b8afdc6fa5e24cbd, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733797494841
2024-12-10T02:24:55,287 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18839f6a2f527eb0eaba0d7ea45ab82a#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:24:55,288 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/f91dad938f3642b98534e6788acee91e is 1080, key is row0002/info:/1733797488779/Put/seqid=0
2024-12-10T02:24:55,290 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:55,290 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35458 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:55,291 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:55,291 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046
2024-12-10T02:24:55,291 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35458 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:55,291 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35458 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35458 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:55,291 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:55,292 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:55,293 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:55,293 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741864_1047
2024-12-10T02:24:55,293 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:55,294 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:55,294 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:55,294 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741865_1048
2024-12-10T02:24:55,295 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:55,296 WARN  [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:55,296 WARN  [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:55,296 WARN  [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741866_1049
2024-12-10T02:24:55,296 WARN  [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:55,297 WARN  [IPC Server handler 0 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:55,297 WARN  [IPC Server handler 0 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:55,297 WARN  [IPC Server handler 0 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:55,300 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741867_1050 (size=17994)
2024-12-10T02:24:55,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4ff31ecb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741847_1030 to 127.0.0.1:40547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:55,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4134f945[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741852_1035 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:55,708 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/f91dad938f3642b98534e6788acee91e as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e
2024-12-10T02:24:55,715 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18839f6a2f527eb0eaba0d7ea45ab82a/info of 18839f6a2f527eb0eaba0d7ea45ab82a into f91dad938f3642b98534e6788acee91e(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:24:55,715 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:55,715 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., storeName=18839f6a2f527eb0eaba0d7ea45ab82a/info, priority=13, startTime=1733797495270; duration=0sec
2024-12-10T02:24:55,715 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T02:24:55,715 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e because midkey is the same as first or last row
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e because midkey is the same as first or last row
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e because midkey is the same as first or last row
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:55,716 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18839f6a2f527eb0eaba0d7ea45ab82a:info
2024-12-10T02:24:56,092 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:56,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T02:24:56,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8ee82486c31d458fa8469486e318cee2 is 1079, key is tmprow/info:/1733797496258/Put/seqid=0
2024-12-10T02:24:56,266 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35486 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051 to mirror 127.0.0.1:40547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,267 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:56,267 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051
2024-12-10T02:24:56,267 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35486 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:56,267 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35486 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35486 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,267 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:56,269 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,269 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35494 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,269 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:56,269 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052
2024-12-10T02:24:56,269 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35494 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:56,269 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35494 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35494 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,270 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:56,271 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46005
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,271 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35510 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053 to mirror 127.0.0.1:46005
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,272 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:56,272 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053
2024-12-10T02:24:56,272 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35510 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:56,272 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35510 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35510 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,272 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:56,274 WARN  [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,274 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35516 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,274 WARN  [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:56,274 WARN  [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054
2024-12-10T02:24:56,274 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35516 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:56,274 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35516 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35516 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,274 WARN  [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:56,275 WARN  [IPC Server handler 4 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:56,275 WARN  [IPC Server handler 4 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:56,275 WARN  [IPC Server handler 4 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:56,278 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741872_1055 (size=6027)
2024-12-10T02:24:56,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4ff31ecb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741862_1045 to 127.0.0.1:40547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4134f945[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741842_1025 to 127.0.0.1:40547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,679 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8ee82486c31d458fa8469486e318cee2
2024-12-10T02:24:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/8ee82486c31d458fa8469486e318cee2 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2
2024-12-10T02:24:56,691 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2, entries=1, sequenceid=45, filesize=5.9 K
2024-12-10T02:24:56,692 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 18839f6a2f527eb0eaba0d7ea45ab82a in 433ms, sequenceid=45, compaction requested=false
2024-12-10T02:24:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-12-10T02:24:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e because midkey is the same as first or last row
2024-12-10T02:24:56,801 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]
2024-12-10T02:24:56,801 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,802 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C40537%2C1733797475089:(num 1733797494787) roll requested
2024-12-10T02:24:56,802 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797496802
2024-12-10T02:24:56,804 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,805 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:56,805 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741873_1056
2024-12-10T02:24:56,805 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:56,806 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,806 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:56,806 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741874_1057
2024-12-10T02:24:56,807 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:56,809 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,809 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:56,809 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35538 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,809 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058
2024-12-10T02:24:56,809 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35538 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T02:24:56,809 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35538 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35538 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,809 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:56,811 WARN  [Thread-963 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:56,811 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35546 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,811 WARN  [Thread-963 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:56,811 WARN  [Thread-963 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059
2024-12-10T02:24:56,811 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35546 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-12-10T02:24:56,812 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35546 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35546 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:56,812 WARN  [Thread-963 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:56,813 WARN  [IPC Server handler 4 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:56,813 WARN  [IPC Server handler 4 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:56,813 WARN  [IPC Server handler 4 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:56,815 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:56,815 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:56,815 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:56,815 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:56,815 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:24:56,816 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797494787 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797496802
2024-12-10T02:24:56,817 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741857_1040 (size=13591)
2024-12-10T02:24:56,820 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44143:44143)]
2024-12-10T02:24:56,820 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:56,821 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797494787 is not closed yet, will try archiving it next time
2024-12-10T02:24:56,821 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797490772 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs/d9f49988d155%2C40537%2C1733797475089.1733797490772
2024-12-10T02:24:57,169 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:57,218 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 is not closed yet, will try archiving it next time
2024-12-10T02:24:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:24:57,676 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T02:24:57,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/a49ab157b2304f5586ff56f34253ab55 is 1079, key is tmprow/info:/1733797497675/Put/seqid=0
2024-12-10T02:24:57,683 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:57,683 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:57,683 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741878_1061
2024-12-10T02:24:57,683 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:57,684 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:57,684 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:57,684 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741879_1062
2024-12-10T02:24:57,685 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:57,686 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:57,686 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:57,686 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741880_1063
2024-12-10T02:24:57,686 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:57,688 WARN  [Thread-969 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:57,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35558 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064 to mirror 127.0.0.1:40547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:57,689 WARN  [Thread-969 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:57,689 WARN  [Thread-969 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064
2024-12-10T02:24:57,689 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35558 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:57,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35558 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35558 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:57,689 WARN  [Thread-969 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:57,690 WARN  [IPC Server handler 2 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:57,690 WARN  [IPC Server handler 2 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:57,690 WARN  [IPC Server handler 2 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:57,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741882_1065 (size=6027)
2024-12-10T02:24:58,093 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,094 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/a49ab157b2304f5586ff56f34253ab55
2024-12-10T02:24:58,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/a49ab157b2304f5586ff56f34253ab55 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55
2024-12-10T02:24:58,105 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55, entries=1, sequenceid=55, filesize=5.9 K
2024-12-10T02:24:58,106 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 18839f6a2f527eb0eaba0d7ea45ab82a in 430ms, sequenceid=55, compaction requested=true
2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e because midkey is the same as first or last row
2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18839f6a2f527eb0eaba0d7ea45ab82a:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:24:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:58,106 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:24:58,107 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:24:58,108 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1541): 18839f6a2f527eb0eaba0d7ea45ab82a/info is initiating minor compaction (all files)
2024-12-10T02:24:58,108 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18839f6a2f527eb0eaba0d7ea45ab82a/info in TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:24:58,108 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55] into tmpdir=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp, totalSize=29.3 K
2024-12-10T02:24:58,108 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91dad938f3642b98534e6788acee91e, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1733797488779
2024-12-10T02:24:58,108 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ee82486c31d458fa8469486e318cee2, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733797496258
2024-12-10T02:24:58,109 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] compactions.Compactor(225): Compacting a49ab157b2304f5586ff56f34253ab55, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1733797497675
2024-12-10T02:24:58,122 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18839f6a2f527eb0eaba0d7ea45ab82a#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:24:58,123 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/7023eba6e20646229d1bf409b0128efe is 1080, key is row0002/info:/1733797488779/Put/seqid=0
2024-12-10T02:24:58,126 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46005
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,126 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35578 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066 to mirror 127.0.0.1:46005
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,126 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:24:58,126 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066
2024-12-10T02:24:58,126 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35578 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:58,126 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35578 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741883_1066] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35578 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,127 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:24:58,128 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,128 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:24:58,128 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741884_1067
2024-12-10T02:24:58,128 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:24:58,130 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,130 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35586 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,130 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:24:58,130 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068
2024-12-10T02:24:58,130 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35586 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:58,130 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35586 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35586 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,131 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:24:58,133 WARN  [Thread-974 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40547
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35588 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069 to mirror 127.0.0.1:40547
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,133 WARN  [Thread-974 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]) is bad.
2024-12-10T02:24:58,133 WARN  [Thread-974 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069
2024-12-10T02:24:58,133 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35588 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:24:58,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:35588 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35588 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,133 WARN  [Thread-974 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40547,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]
2024-12-10T02:24:58,134 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-12-10T02:24:58,134 WARN  [IPC Server handler 3 on default port 42523 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-12-10T02:24:58,134 WARN  [IPC Server handler 3 on default port 42523 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-12-10T02:24:58,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741887_1070 (size=18097)
2024-12-10T02:24:58,544 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/7023eba6e20646229d1bf409b0128efe as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/7023eba6e20646229d1bf409b0128efe
2024-12-10T02:24:58,551 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18839f6a2f527eb0eaba0d7ea45ab82a/info of 18839f6a2f527eb0eaba0d7ea45ab82a into 7023eba6e20646229d1bf409b0128efe(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:24:58,552 INFO  [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., storeName=18839f6a2f527eb0eaba0d7ea45ab82a/info, priority=13, startTime=1733797498106; duration=0sec
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/7023eba6e20646229d1bf409b0128efe because midkey is the same as first or last row
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/7023eba6e20646229d1bf409b0128efe because midkey is the same as first or last row
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/7023eba6e20646229d1bf409b0128efe because midkey is the same as first or last row
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:24:58,552 DEBUG [RS:0;d9f49988d155:40537-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18839f6a2f527eb0eaba0d7ea45ab82a:info
2024-12-10T02:24:58,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4ff31ecb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741872_1055 to 127.0.0.1:40547 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,582 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4134f945[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741867_1050 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:58,821 WARN  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-12-10T02:24:58,821 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:58,897 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:24:58,900 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:24:58,902 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:24:58,902 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:24:58,902 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:24:58,902 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@719d00c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:24:58,903 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc86a20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:24:59,017 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@402253c7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/java.io.tmpdir/jetty-localhost-40071-hadoop-hdfs-3_4_1-tests_jar-_-any-15042088895817970781/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:24:59,018 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5dd0b56c{HTTP/1.1, (http/1.1)}{localhost:40071}
2024-12-10T02:24:59,018 INFO  [Time-limited test {}] server.Server(415): Started @128076ms
2024-12-10T02:24:59,019 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:24:59,118 WARN  [Thread-995 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:24:59,126 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x127868d936e1aed3 with lease ID 0x40f35a92553cf619: from storage DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0 node DatanodeRegistration(127.0.0.1:36499, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=33775, infoSecurePort=0, ipcPort=36371, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:59,126 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x127868d936e1aed3 with lease ID 0x40f35a92553cf619: from storage DS-37337454-7f71-4c30-add1-ee1eabb6911b node DatanodeRegistration(127.0.0.1:36499, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=33775, infoSecurePort=0, ipcPort=36371, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:24:59,170 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:24:59,583 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4ff31ecb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741882_1065 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:24:59,586 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741857_1040 (size=13591)
2024-12-10T02:25:00,093 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:00,821 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:01,170 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:01,583 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4134f945[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741887_1070 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:02,093 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:02,822 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:03,171 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:04,094 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:04,822 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,020 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:25:05,171 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,274 ERROR [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData-prefix:d9f49988d155,44417,1733797475040 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,274 WARN  [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData-prefix:d9f49988d155,44417,1733797475040 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,275 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C44417%2C1733797475040:(num 1733797475187) roll requested
2024-12-10T02:25:05,275 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C44417%2C1733797475040.1733797505275
2024-12-10T02:25:05,278 WARN  [Thread-1016 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,278 WARN  [Thread-1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:36499,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:05,278 WARN  [Thread-1016 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741888_1071
2024-12-10T02:25:05,279 WARN  [Thread-1016 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:05,280 WARN  [Thread-1016 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1072
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,280 WARN  [Thread-1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741889_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:05,280 WARN  [Thread-1016 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741889_1072
2024-12-10T02:25:05,281 WARN  [Thread-1016 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:05,285 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:05,286 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:05,286 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:05,286 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:05,286 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:05,286 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 with entries=54, filesize=26.65 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797505275
2024-12-10T02:25:05,286 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,287 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:05,287 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187
2024-12-10T02:25:05,287 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44143:44143),(127.0.0.1/127.0.0.1:33775:33775)]
2024-12-10T02:25:05,287 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 is not closed yet, will try archiving it next time
2024-12-10T02:25:05,287 WARN  [IPC Server handler 2 on default port 42523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 has not been closed. Lease recovery is in progress. RecoveryId = 1074 for block blk_1073741830_1006
2024-12-10T02:25:05,287 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 after 0ms
2024-12-10T02:25:06,094 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:06,822 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:08,094 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:08,823 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:09,139 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@36149595 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:46005,null,null])
java.net.ConnectException: Call From d9f49988d155/172.17.0.2 to localhost:36479 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-10T02:25:09,141 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741833_1020 (size=455)
2024-12-10T02:25:09,289 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 after 4002ms
2024-12-10T02:25:09,794 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797475506 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs/d9f49988d155%2C40537%2C1733797475089.1733797475506
2024-12-10T02:25:09,796 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797494787 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs/d9f49988d155%2C40537%2C1733797475089.1733797494787
2024-12-10T02:25:10,095 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:10,823 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:11,123 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@7e84013[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36499, datanodeUuid=2dc758ca-efcb-43af-ab9b-dd6e6afe3034, infoPort=33775, infoSecurePort=0, ipcPort=36371, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741833_1020 to 127.0.0.1:40783 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:12,095 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,741 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.1733797512741
2024-12-10T02:25:12,744 WARN  [Thread-1026 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1075
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,744 WARN  [Thread-1026 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741891_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:12,744 WARN  [Thread-1026 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741891_1075
2024-12-10T02:25:12,745 WARN  [Thread-1026 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:12,749 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,749 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,749 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,749 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,749 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,750 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797496802 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.1733797512741
2024-12-10T02:25:12,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741877_1060 (size=12911)
2024-12-10T02:25:12,752 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44143:44143),(127.0.0.1/127.0.0.1:33775:33775)]
2024-12-10T02:25:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40537 {}] regionserver.HRegion(8855): Flush requested on 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:25:12,757 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-12-10T02:25:12,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/02f08c4d1e304bffb1fffd770ecaa263 is 1080, key is row0013/info:/1733797512754/Put/seqid=0
2024-12-10T02:25:12,763 WARN  [Thread-1032 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,764 WARN  [Thread-1032 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:12,764 WARN  [Thread-1032 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741893_1077
2024-12-10T02:25:12,764 WARN  [Thread-1032 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:12,767 WARN  [Thread-1032 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48082 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data4]'}, localName='127.0.0.1:36499', datanodeUuid='2dc758ca-efcb-43af-ab9b-dd6e6afe3034', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:12,767 WARN  [Thread-1032 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36499,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:12,767 WARN  [Thread-1032 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078
2024-12-10T02:25:12,767 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48082 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078] {}] datanode.BlockReceiver(316): Block 1073741894 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:25:12,767 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48082 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741894_1078] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48082 dst: /127.0.0.1:36499
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:12,768 WARN  [Thread-1032 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:12,769 WARN  [Thread-1032 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,769 WARN  [Thread-1032 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:25:12,769 WARN  [Thread-1032 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741895_1079
2024-12-10T02:25:12,769 WARN  [Thread-1032 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:25:12,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741896_1080 (size=8190)
2024-12-10T02:25:12,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741896_1080 (size=8190)
2024-12-10T02:25:12,776 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/02f08c4d1e304bffb1fffd770ecaa263
2024-12-10T02:25:12,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/02f08c4d1e304bffb1fffd770ecaa263 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/02f08c4d1e304bffb1fffd770ecaa263
2024-12-10T02:25:12,788 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/02f08c4d1e304bffb1fffd770ecaa263, entries=3, sequenceid=66, filesize=8.0 K
2024-12-10T02:25:12,789 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 18839f6a2f527eb0eaba0d7ea45ab82a in 32ms, sequenceid=66, compaction requested=false
2024-12-10T02:25:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18839f6a2f527eb0eaba0d7ea45ab82a:

2024-12-10T02:25:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-12-10T02:25:12,789 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:25:12,790 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/7023eba6e20646229d1bf409b0128efe because midkey is the same as first or last row
2024-12-10T02:25:12,823 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-12-10T02:25:12,823 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,975 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:25:12,975 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:25:12,975 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:12,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:12,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:12,975 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:25:12,975 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:25:12,975 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1048628396, stopped=false
2024-12-10T02:25:12,976 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,44417,1733797475040
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:12,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:25:12,978 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:12,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,40537,1733797475089' *****
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,33309,1733797476037' *****
2024-12-10T02:25:12,978 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:25:12,979 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:12,979 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:25:12,979 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:12,979 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:25:12,979 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:25:12,979 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:25:12,979 INFO  [RS:0;d9f49988d155:40537 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,33309,1733797476037
2024-12-10T02:25:12,979 INFO  [RS:0;d9f49988d155:40537 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:25:12,979 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(3091): Received CLOSE for 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:25:12,979 INFO  [RS:1;d9f49988d155:33309 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d9f49988d155:33309.
2024-12-10T02:25:12,979 DEBUG [RS:1;d9f49988d155:33309 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:12,979 DEBUG [RS:1;d9f49988d155:33309 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:12,980 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,33309,1733797476037; all regions closed.
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,40537,1733797475089
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:40537.
2024-12-10T02:25:12,980 DEBUG [RS:0;d9f49988d155:40537 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:12,980 DEBUG [RS:0;d9f49988d155:40537 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:25:12,980 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:25:12,980 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 18839f6a2f527eb0eaba0d7ea45ab82a, disabling compactions & flushes
2024-12-10T02:25:12,980 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:25:12,980 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,980 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:25:12,980 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a. after waiting 0 ms
2024-12-10T02:25:12,980 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:25:12,980 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,980 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,980 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T02:25:12,980 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 18839f6a2f527eb0eaba0d7ea45ab82a 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-12-10T02:25:12,980 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1325): Online Regions={18839f6a2f527eb0eaba0d7ea45ab82a=TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., 1588230740=hbase:meta,,1.1588230740}
2024-12-10T02:25:12,981 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,981 DEBUG [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 18839f6a2f527eb0eaba0d7ea45ab82a
2024-12-10T02:25:12,981 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:25:12,981 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:25:12,981 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:25:12,981 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:25:12,981 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:25:12,981 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-12-10T02:25:12,981 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,981 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,981 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
2024-12-10T02:25:12,981 ERROR [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e-prefix:d9f49988d155,40537,1733797475089.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,981 WARN  [FSHLog-0-hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e-prefix:d9f49988d155,40537,1733797475089.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,982 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C40537%2C1733797475089.meta:.meta(num 1733797475892) roll requested
2024-12-10T02:25:12,982 WARN  [IPC Server handler 2 on default port 42523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741837_1013
2024-12-10T02:25:12,982 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40537%2C1733797475089.meta.1733797512982.meta
2024-12-10T02:25:12,982 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 after 1ms
2024-12-10T02:25:12,985 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1082
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,986 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741897_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:12,986 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741897_1082
2024-12-10T02:25:12,986 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:12,987 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/69fd7748411a46208917a19e7f360002 is 1080, key is row0015/info:/1733797512758/Put/seqid=0
2024-12-10T02:25:12,987 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1083
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,987 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741898_1083 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]) is bad.
2024-12-10T02:25:12,987 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741898_1083
2024-12-10T02:25:12,988 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]
2024-12-10T02:25:12,988 WARN  [Thread-1042 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1084
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,988 WARN  [Thread-1042 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741899_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:12,988 WARN  [Thread-1042 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741899_1084
2024-12-10T02:25:12,989 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1085
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,989 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741900_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK], DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:12,989 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741900_1085
2024-12-10T02:25:12,989 WARN  [Thread-1042 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:12,989 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:12,990 WARN  [Thread-1042 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1086
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:12,990 WARN  [Thread-1042 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741901_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK], DatanodeInfoWithStorage[127.0.0.1:36499,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:12,990 WARN  [Thread-1042 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741901_1086
2024-12-10T02:25:12,991 WARN  [Thread-1042 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:12,996 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,996 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,997 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,997 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,997 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:12,997 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797512982.meta
2024-12-10T02:25:13,000 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,000 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:46005,DS-be6d402c-06f6-4868-8fa2-34f6c2505937,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,000 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
2024-12-10T02:25:13,001 WARN  [IPC Server handler 0 on default port 42523 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta has not been closed. Lease recovery is in progress. RecoveryId = 1089 for block blk_1073741834_1010
2024-12-10T02:25:13,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741903_1088 (size=14660)
2024-12-10T02:25:13,001 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta after 1ms
2024-12-10T02:25:13,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741903_1088 (size=14660)
2024-12-10T02:25:13,001 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/69fd7748411a46208917a19e7f360002
2024-12-10T02:25:13,004 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44143:44143),(127.0.0.1/127.0.0.1:33775:33775)]
2024-12-10T02:25:13,004 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta is not closed yet, will try archiving it next time
2024-12-10T02:25:13,008 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/.tmp/info/69fd7748411a46208917a19e7f360002 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/69fd7748411a46208917a19e7f360002
2024-12-10T02:25:13,013 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/69fd7748411a46208917a19e7f360002, entries=9, sequenceid=78, filesize=14.3 K
2024-12-10T02:25:13,014 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 18839f6a2f527eb0eaba0d7ea45ab82a in 34ms, sequenceid=78, compaction requested=true
2024-12-10T02:25:13,015 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55] to archive
2024-12-10T02:25:13,016 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:25:13,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/fc3a2ef7a5d94e558f7a8b8034c92c05
2024-12-10T02:25:13,018 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8955215aab74460cb983a2388e307c69
2024-12-10T02:25:13,022 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/f91dad938f3642b98534e6788acee91e
2024-12-10T02:25:13,022 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/1de715bdef8e4d61b8afdc6fa5e24cbd
2024-12-10T02:25:13,022 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/8ee82486c31d458fa8469486e318cee2
2024-12-10T02:25:13,022 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/info/a49ab157b2304f5586ff56f34253ab55
2024-12-10T02:25:13,023 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d9f49988d155:44417 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T02:25:13,023 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [fc3a2ef7a5d94e558f7a8b8034c92c05=10347, 8955215aab74460cb983a2388e307c69=12506, f91dad938f3642b98534e6788acee91e=17994, 1de715bdef8e4d61b8afdc6fa5e24cbd=6027, 8ee82486c31d458fa8469486e318cee2=6027, a49ab157b2304f5586ff56f34253ab55=6027]
2024-12-10T02:25:13,027 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/default/TestLogRolling-testLogRollOnDatanodeDeath/18839f6a2f527eb0eaba0d7ea45ab82a/recovered.edits/81.seqid, newMaxSeqId=81, maxSeqId=1
2024-12-10T02:25:13,028 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:25:13,028 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 18839f6a2f527eb0eaba0d7ea45ab82a:
Waiting for close lock at 1733797512980Running coprocessor pre-close hooks at 1733797512980Disabling compacts and flushes for region at 1733797512980Disabling writes for close at 1733797512980Obtaining lock to block concurrent updates at 1733797512980Preparing flush snapshotting stores in 18839f6a2f527eb0eaba0d7ea45ab82a at 1733797512980Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a., syncing WAL and waiting on mvcc, flushsize=dataSize=9683, getHeapSize=10608, getOffHeapSize=0, getCellsCount=9 at 1733797512981 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a. at 1733797512982 (+1 ms)Flushing 18839f6a2f527eb0eaba0d7ea45ab82a/info: creating writer at 1733797512982Flushing 18839f6a2f527eb0eaba0d7ea45ab82a/info: appending metadata at 1733797512986 (+4 ms)Flushing 18839f6a2f527eb0eaba0d7ea45ab82a/info: closing flushed file at 1733797512986Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@185b5646: reopening flushed file at 1733797513007 (+21 ms)Finished flush of dataSize ~9.46 KB/9683, heapSize ~10.36 KB/10608, currentSize=0 B/0 for 18839f6a2f527eb0eaba0d7ea45ab82a in 34ms, sequenceid=78, compaction requested=true at 1733797513014 (+7 ms)Writing region close event to WAL at 1733797513024 (+10 ms)Running coprocessor post-close hooks at 1733797513028 (+4 ms)Closed at 1733797513028
2024-12-10T02:25:13,028 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a.
2024-12-10T02:25:13,028 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/info/84d42e2050fb4d5d8cee63fa245c24ea is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1733797476124.18839f6a2f527eb0eaba0d7ea45ab82a./info:regioninfo/1733797476482/Put/seqid=0
2024-12-10T02:25:13,031 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1090
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,031 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50038 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,031 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:13,031 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50038 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090] {}] datanode.BlockReceiver(316): Block 1073741904 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:25:13,031 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090
2024-12-10T02:25:13,031 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50038 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741904_1090] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50038 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,032 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:13,034 WARN  [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741905_1091
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50052 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,034 WARN  [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:13,034 WARN  [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091
2024-12-10T02:25:13,034 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50052 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091] {}] datanode.BlockReceiver(316): Block 1073741905 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:25:13,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50052 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741905_1091] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50052 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,034 WARN  [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:13,038 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741906_1092 (size=7089)
2024-12-10T02:25:13,039 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741906_1092 (size=7089)
2024-12-10T02:25:13,039 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/info/84d42e2050fb4d5d8cee63fa245c24ea
2024-12-10T02:25:13,060 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/ns/d0248776cef944e589ade532b3ef050d is 43, key is default/ns:d/1733797475966/Put/seqid=0
2024-12-10T02:25:13,062 WARN  [Thread-1062 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1093
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:38163
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,062 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50078 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8]'}, localName='127.0.0.1:44705', datanodeUuid='66a55661-c1bc-4a03-adeb-847970538236', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093 to mirror 127.0.0.1:38163
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,063 WARN  [Thread-1062 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44705,DS-78ed481c-1d9e-48b8-8798-1e4695865a7f,DISK], DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]) is bad.
2024-12-10T02:25:13,063 WARN  [Thread-1062 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093
2024-12-10T02:25:13,063 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50078 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093] {}] datanode.BlockReceiver(316): Block 1073741907 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:25:13,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:50078 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741907_1093] {}] datanode.DataXceiver(331): 127.0.0.1:44705:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50078 dst: /127.0.0.1:44705
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,063 WARN  [Thread-1062 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:38163,DS-96dda52a-6590-41f0-8054-8f67479a0831,DISK]
2024-12-10T02:25:13,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741908_1094 (size=5153)
2024-12-10T02:25:13,068 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741908_1094 (size=5153)
2024-12-10T02:25:13,068 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/ns/d0248776cef944e589ade532b3ef050d
2024-12-10T02:25:13,089 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/table/bf6e7fe0d09c4744969da124c997ef17 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1733797476492/Put/seqid=0
2024-12-10T02:25:13,092 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741909_1095
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40783
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:13,092 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48156 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data4]'}, localName='127.0.0.1:36499', datanodeUuid='2dc758ca-efcb-43af-ab9b-dd6e6afe3034', xmitsInProgress=0}:Exception transferring block BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095 to mirror 127.0.0.1:40783
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,092 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36499,DS-25403030-5fc0-4f7c-8c6f-ee61180e45a0,DISK], DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]) is bad.
2024-12-10T02:25:13,092 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095
2024-12-10T02:25:13,092 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48156 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095] {}] datanode.BlockReceiver(316): Block 1073741909 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-12-10T02:25:13,092 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1401978279_22 at /127.0.0.1:48156 [Receiving block BP-35725305-172.17.0.2-1733797474215:blk_1073741909_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36499:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48156 dst: /127.0.0.1:36499
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:13,093 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40783,DS-de2d2872-87b8-46e9-bed7-6799d4c0ad91,DISK]
2024-12-10T02:25:13,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741910_1096 (size=5424)
2024-12-10T02:25:13,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741910_1096 (size=5424)
2024-12-10T02:25:13,097 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/table/bf6e7fe0d09c4744969da124c997ef17
2024-12-10T02:25:13,104 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/info/84d42e2050fb4d5d8cee63fa245c24ea as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/info/84d42e2050fb4d5d8cee63fa245c24ea
2024-12-10T02:25:13,111 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/info/84d42e2050fb4d5d8cee63fa245c24ea, entries=10, sequenceid=11, filesize=6.9 K
2024-12-10T02:25:13,112 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/ns/d0248776cef944e589ade532b3ef050d as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/ns/d0248776cef944e589ade532b3ef050d
2024-12-10T02:25:13,118 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/ns/d0248776cef944e589ade532b3ef050d, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T02:25:13,119 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/.tmp/table/bf6e7fe0d09c4744969da124c997ef17 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/table/bf6e7fe0d09c4744969da124c997ef17
2024-12-10T02:25:13,125 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/table/bf6e7fe0d09c4744969da124c997ef17, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T02:25:13,126 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 145ms, sequenceid=11, compaction requested=false
2024-12-10T02:25:13,131 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T02:25:13,132 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:13,132 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:13,132 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797512981Running coprocessor pre-close hooks at 1733797512981Disabling compacts and flushes for region at 1733797512981Disabling writes for close at 1733797512981Obtaining lock to block concurrent updates at 1733797512981Preparing flush snapshotting stores in 1588230740 at 1733797512981Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1733797512981Flushing stores of hbase:meta,,1.1588230740 at 1733797513006 (+25 ms)Flushing 1588230740/info: creating writer at 1733797513006Flushing 1588230740/info: appending metadata at 1733797513028 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733797513028Flushing 1588230740/ns: creating writer at 1733797513045 (+17 ms)Flushing 1588230740/ns: appending metadata at 1733797513060 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733797513060Flushing 1588230740/table: creating writer at 1733797513074 (+14 ms)Flushing 1588230740/table: appending metadata at 1733797513089 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733797513089Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@367d5ac4: reopening flushed file at 1733797513104 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69ee9114: reopening flushed file at 1733797513111 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fb77576: reopening flushed file at 1733797513118 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 145ms, sequenceid=11, compaction requested=false at 1733797513126 (+8 ms)Writing region close event to WAL at 1733797513128 (+2 ms)Running coprocessor post-close hooks at 1733797513132 (+4 ms)Closed at 1733797513132
2024-12-10T02:25:13,132 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:13,154 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T02:25:13,154 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T02:25:13,181 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,40537,1733797475089; all regions closed.
2024-12-10T02:25:13,181 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:13,181 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:13,181 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:13,182 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:13,182 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:13,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741902_1087 (size=825)
2024-12-10T02:25:13,184 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741902_1087 (size=825)
2024-12-10T02:25:13,360 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T02:25:13,360 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T02:25:13,361 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:13,583 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4134f945[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44705, datanodeUuid=66a55661-c1bc-4a03-adeb-847970538236, infoPort=44143, infoSecurePort=0, ipcPort=42195, storageInfo=lv=-57;cid=testClusterID;nsid=147446996;c=1733797474215):Failed to transfer BP-35725305-172.17.0.2-1733797474215:blk_1073741877_1060 to 127.0.0.1:46005 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:14,093 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:16,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-12-10T02:25:16,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:16,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:25:16,466 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T02:25:16,466 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T02:25:16,983 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 after 4002ms
2024-12-10T02:25:17,002 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta after 4002ms
2024-12-10T02:25:17,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:25:17,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741836_1012 (size=76)
2024-12-10T02:25:17,981 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-10T02:25:17,983 DEBUG [RS:1;d9f49988d155:33309 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs
2024-12-10T02:25:17,983 INFO  [RS:1;d9f49988d155:33309 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C33309%2C1733797476037:(num 1733797476224)
2024-12-10T02:25:17,983 DEBUG [RS:1;d9f49988d155:33309 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:17,983 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:25:17,984 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:25:17,984 INFO  [RS:1;d9f49988d155:33309 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33309
2024-12-10T02:25:17,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:25:17,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,33309,1733797476037
2024-12-10T02:25:17,986 INFO  [RS:1;d9f49988d155:33309 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:25:17,988 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor115.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:17,989 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,33309,1733797476037]
2024-12-10T02:25:17,990 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,33309,1733797476037 already deleted, retry=false
2024-12-10T02:25:17,990 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,33309,1733797476037 expired; onlineServers=1
2024-12-10T02:25:18,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,044 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,051 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,052 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:18,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33309-0x1019a2fdd5e0002, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:18,089 INFO  [RS:1;d9f49988d155:33309 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:25:18,089 INFO  [RS:1;d9f49988d155:33309 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,33309,1733797476037; zookeeper connection closed.
2024-12-10T02:25:18,089 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c44c8c7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c44c8c7
2024-12-10T02:25:18,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:25:18,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:25:18,182 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-12-10T02:25:18,185 DEBUG [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs
2024-12-10T02:25:18,185 INFO  [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C40537%2C1733797475089.meta:.meta(num 1733797512982)
2024-12-10T02:25:18,186 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,186 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,186 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,186 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,186 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741892_1076 (size=14682)
2024-12-10T02:25:18,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741892_1076 (size=14682)
2024-12-10T02:25:18,191 DEBUG [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs
2024-12-10T02:25:18,191 INFO  [RS:0;d9f49988d155:40537 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C40537%2C1733797475089:(num 1733797512741)
2024-12-10T02:25:18,191 DEBUG [RS:0;d9f49988d155:40537 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:18,191 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:18,191 INFO  [RS:0;d9f49988d155:40537 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:25:18,192 INFO  [RS:0;d9f49988d155:40537 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T02:25:18,192 INFO  [RS:0;d9f49988d155:40537 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:25:18,192 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:25:18,192 INFO  [RS:0;d9f49988d155:40537 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40537
2024-12-10T02:25:18,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:25:18,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,40537,1733797475089
2024-12-10T02:25:18,194 INFO  [RS:0;d9f49988d155:40537 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:25:18,195 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,40537,1733797475089]
2024-12-10T02:25:18,198 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,40537,1733797475089 already deleted, retry=false
2024-12-10T02:25:18,198 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,40537,1733797475089 expired; onlineServers=0
2024-12-10T02:25:18,198 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,44417,1733797475040' *****
2024-12-10T02:25:18,198 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:25:18,198 INFO  [M:0;d9f49988d155:44417 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:25:18,198 INFO  [M:0;d9f49988d155:44417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:25:18,198 DEBUG [M:0;d9f49988d155:44417 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:25:18,198 DEBUG [M:0;d9f49988d155:44417 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:25:18,198 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:25:18,198 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797475274 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797475274,5,FailOnTimeoutGroup]
2024-12-10T02:25:18,198 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797475275 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797475275,5,FailOnTimeoutGroup]
2024-12-10T02:25:18,199 INFO  [M:0;d9f49988d155:44417 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:25:18,199 INFO  [M:0;d9f49988d155:44417 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:25:18,199 DEBUG [M:0;d9f49988d155:44417 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:25:18,199 INFO  [M:0;d9f49988d155:44417 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:25:18,199 INFO  [M:0;d9f49988d155:44417 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:25:18,199 INFO  [M:0;d9f49988d155:44417 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:25:18,199 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:25:18,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:25:18,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:18,200 DEBUG [M:0;d9f49988d155:44417 {}] zookeeper.ZKUtil(347): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:25:18,200 WARN  [M:0;d9f49988d155:44417 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:25:18,201 INFO  [M:0;d9f49988d155:44417 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/.lastflushedseqids
2024-12-10T02:25:18,206 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741911_1097 (size=130)
2024-12-10T02:25:18,207 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741911_1097 (size=130)
2024-12-10T02:25:18,207 INFO  [M:0;d9f49988d155:44417 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:25:18,208 INFO  [M:0;d9f49988d155:44417 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:25:18,208 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:25:18,208 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:18,208 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:18,208 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:25:18,208 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:18,208 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.24 KB heapSize=29.47 KB
2024-12-10T02:25:18,224 DEBUG [M:0;d9f49988d155:44417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15393d607d04b7eba6697e7a8d44fd3 is 82, key is hbase:meta,,1/info:regioninfo/1733797475948/Put/seqid=0
2024-12-10T02:25:18,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741912_1098 (size=5672)
2024-12-10T02:25:18,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741912_1098 (size=5672)
2024-12-10T02:25:18,230 INFO  [M:0;d9f49988d155:44417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15393d607d04b7eba6697e7a8d44fd3
2024-12-10T02:25:18,258 DEBUG [M:0;d9f49988d155:44417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/566ffc2d569649239ba2f87084ff8975 is 773, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733797476497/Put/seqid=0
2024-12-10T02:25:18,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741913_1099 (size=6254)
2024-12-10T02:25:18,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741913_1099 (size=6254)
2024-12-10T02:25:18,264 INFO  [M:0;d9f49988d155:44417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/566ffc2d569649239ba2f87084ff8975
2024-12-10T02:25:18,269 INFO  [M:0;d9f49988d155:44417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 566ffc2d569649239ba2f87084ff8975
2024-12-10T02:25:18,284 DEBUG [M:0;d9f49988d155:44417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea65d5856be34f429a8d8dc15b9ec692 is 69, key is d9f49988d155,33309,1733797476037/rs:state/1733797476078/Put/seqid=0
2024-12-10T02:25:18,289 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741914_1100 (size=5224)
2024-12-10T02:25:18,290 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741914_1100 (size=5224)
2024-12-10T02:25:18,290 INFO  [M:0;d9f49988d155:44417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea65d5856be34f429a8d8dc15b9ec692
2024-12-10T02:25:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:18,295 INFO  [RS:0;d9f49988d155:40537 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:25:18,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40537-0x1019a2fdd5e0001, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:18,295 INFO  [RS:0;d9f49988d155:40537 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,40537,1733797475089; zookeeper connection closed.
2024-12-10T02:25:18,296 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@45f3bf9f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@45f3bf9f
2024-12-10T02:25:18,296 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-10T02:25:18,310 DEBUG [M:0;d9f49988d155:44417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1eb2f16df830407ab2711b08cdf84892 is 52, key is load_balancer_on/state:d/1733797476020/Put/seqid=0
2024-12-10T02:25:18,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741915_1101 (size=5056)
2024-12-10T02:25:18,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741915_1101 (size=5056)
2024-12-10T02:25:18,316 INFO  [M:0;d9f49988d155:44417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1eb2f16df830407ab2711b08cdf84892
2024-12-10T02:25:18,321 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d15393d607d04b7eba6697e7a8d44fd3 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d15393d607d04b7eba6697e7a8d44fd3
2024-12-10T02:25:18,326 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d15393d607d04b7eba6697e7a8d44fd3, entries=8, sequenceid=60, filesize=5.5 K
2024-12-10T02:25:18,327 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/566ffc2d569649239ba2f87084ff8975 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/566ffc2d569649239ba2f87084ff8975
2024-12-10T02:25:18,331 INFO  [M:0;d9f49988d155:44417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 566ffc2d569649239ba2f87084ff8975
2024-12-10T02:25:18,331 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/566ffc2d569649239ba2f87084ff8975, entries=6, sequenceid=60, filesize=6.1 K
2024-12-10T02:25:18,332 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ea65d5856be34f429a8d8dc15b9ec692 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ea65d5856be34f429a8d8dc15b9ec692
2024-12-10T02:25:18,337 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ea65d5856be34f429a8d8dc15b9ec692, entries=2, sequenceid=60, filesize=5.1 K
2024-12-10T02:25:18,338 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1eb2f16df830407ab2711b08cdf84892 as hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1eb2f16df830407ab2711b08cdf84892
2024-12-10T02:25:18,343 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1eb2f16df830407ab2711b08cdf84892, entries=1, sequenceid=60, filesize=4.9 K
2024-12-10T02:25:18,344 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false
2024-12-10T02:25:18,345 INFO  [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:18,345 DEBUG [M:0;d9f49988d155:44417 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797518208Disabling compacts and flushes for region at 1733797518208Disabling writes for close at 1733797518208Obtaining lock to block concurrent updates at 1733797518208Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797518208Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23793, getHeapSize=30112, getOffHeapSize=0, getCellsCount=71 at 1733797518209 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797518209Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797518209Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797518224 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797518224Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797518236 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797518257 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797518257Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797518269 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797518284 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797518284Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797518295 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797518310 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797518310Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d7b09a4: reopening flushed file at 1733797518320 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@af33cee: reopening flushed file at 1733797518326 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f92b3c9: reopening flushed file at 1733797518331 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@462e607c: reopening flushed file at 1733797518337 (+6 ms)Finished flush of dataSize ~23.24 KB/23793, heapSize ~29.41 KB/30112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=60, compaction requested=false at 1733797518344 (+7 ms)Writing region close event to WAL at 1733797518345 (+1 ms)Closed at 1733797518345
2024-12-10T02:25:18,346 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,346 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,346 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,346 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,346 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:18,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36499 is added to blk_1073741890_1073 (size=1045)
2024-12-10T02:25:18,348 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44705 is added to blk_1073741890_1073 (size=1045)
2024-12-10T02:25:18,554 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:25:18,569 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,569 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,570 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,570 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,571 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,571 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,577 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,580 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:18,989 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:19,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:19,143 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c876cc2 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:46005,null,null])
java.net.ConnectException: Call From d9f49988d155/172.17.0.2 to localhost:36479 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-10T02:25:19,296 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/WALs/d9f49988d155,44417,1733797475040/d9f49988d155%2C44417%2C1733797475040.1733797475187 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/oldWALs/d9f49988d155%2C44417%2C1733797475040.1733797475187
2024-12-10T02:25:19,299 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/MasterData/oldWALs/d9f49988d155%2C44417%2C1733797475040.1733797475187 to hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/oldWALs/d9f49988d155%2C44417%2C1733797475040.1733797475187$masterlocalwal$
2024-12-10T02:25:19,299 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:25:19,299 INFO  [M:0;d9f49988d155:44417 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:25:19,299 INFO  [M:0;d9f49988d155:44417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44417
2024-12-10T02:25:19,300 INFO  [M:0;d9f49988d155:44417 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:25:19,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:19,401 INFO  [M:0;d9f49988d155:44417 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:25:19,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44417-0x1019a2fdd5e0000, quorum=127.0.0.1:60905, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:19,404 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@402253c7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:19,404 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5dd0b56c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:19,404 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:19,405 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc86a20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:19,405 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@719d00c0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:19,406 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:19,406 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:19,406 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:19,406 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid 2dc758ca-efcb-43af-ab9b-dd6e6afe3034) service to localhost/127.0.0.1:42523
2024-12-10T02:25:19,406 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:46005,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:36479 , LocalHost:localPort d9f49988d155/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-12-10T02:25:19,407 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:36499,null,null])
java.io.IOException: No block pool offer service for bpid=BP-35725305-172.17.0.2-1733797474215
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:19,407 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:46005,null,null], DatanodeInfoWithStorage[127.0.0.1:36499,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-35725305-172.17.0.2-1733797474215:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:46005,null,null], DatanodeInfoWithStorage[127.0.0.1:36499,null,null]]
2024-12-10T02:25:19,407 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data3/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:19,407 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:46005,null,null])
java.io.IOException: No block pool offer service for bpid=BP-35725305-172.17.0.2-1733797474215
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:19,407 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:36499,null,null])
java.io.IOException: No block pool offer service for bpid=BP-35725305-172.17.0.2-1733797474215
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:19,407 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@4491e7e0 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:46005,null,null], DatanodeInfoWithStorage[127.0.0.1:36499,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-35725305-172.17.0.2-1733797474215:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:46005,null,null], DatanodeInfoWithStorage[127.0.0.1:36499,null,null]]
2024-12-10T02:25:19,407 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data4/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:19,408 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:19,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@39835cdd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:19,410 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@12b19204{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:19,410 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:19,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3afd309b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:19,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@712f5f14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:19,411 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:19,411 WARN  [BP-35725305-172.17.0.2-1733797474215 heartbeating to localhost/127.0.0.1:42523 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-35725305-172.17.0.2-1733797474215 (Datanode Uuid 66a55661-c1bc-4a03-adeb-847970538236) service to localhost/127.0.0.1:42523
2024-12-10T02:25:19,411 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:19,411 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:19,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data7/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:19,412 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/cluster_073aec79-6014-53e8-e4e1-fe5f68e75c7d/data/data8/current/BP-35725305-172.17.0.2-1733797474215 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:19,412 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:19,419 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cd2a640{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:25:19,419 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@64b7b556{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:19,419 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:19,419 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ed3a961{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:19,420 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a488aac{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:19,428 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:25:19,455 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:25:19,463 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=162 (was 84)
Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42523 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HFileArchiver-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-8
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42523
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42523 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:44885
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42523
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42523
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:42523
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f488cbf4b78.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44885
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42523
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42523 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42523
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$899/0x00007f488cbf4b78.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-7
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42523 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:42523
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-9
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=448 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=27 (was 48), ProcessCount=11 (was 11), AvailableMemoryMB=3926 (was 4371)
2024-12-10T02:25:19,470 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=162, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=27, ProcessCount=11, AvailableMemoryMB=3926
2024-12-10T02:25:19,470 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.log.dir so I do NOT create it in target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/21f69832-ec76-06fe-474e-3792d57936ce/hadoop.tmp.dir so I do NOT create it in target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c, deleteOnExit=true
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/test.cache.data in system properties and HBase conf
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:25:19,471 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:25:19,471 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:25:19,472 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:25:19,473 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:25:19,486 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:25:19,558 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:19,562 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:19,564 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:19,564 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:19,564 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:19,565 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:19,565 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:19,565 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:19,679 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16304a50{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-36169-hadoop-hdfs-3_4_1-tests_jar-_-any-7229358610274568692/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:25:19,680 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:36169}
2024-12-10T02:25:19,680 INFO  [Time-limited test {}] server.Server(415): Started @148738ms
2024-12-10T02:25:19,693 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:25:19,768 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:19,771 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:19,773 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:19,773 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:19,773 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:25:19,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:19,774 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:19,889 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21865735{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-42717-hadoop-hdfs-3_4_1-tests_jar-_-any-2384302793942639793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:19,890 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:42717}
2024-12-10T02:25:19,890 INFO  [Time-limited test {}] server.Server(415): Started @148948ms
2024-12-10T02:25:19,892 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:19,921 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:19,924 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:19,925 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:19,925 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:19,925 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:25:19,926 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:19,926 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:19,986 WARN  [Thread-1191 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data1/current/BP-1457422663-172.17.0.2-1733797519504/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:19,986 WARN  [Thread-1192 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data2/current/BP-1457422663-172.17.0.2-1733797519504/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:19,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:20,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:20,007 WARN  [Thread-1170 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:20,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43a3d75df02a3774 with lease ID 0x2c1a4f584b5ecf3a: Processing first storage report for DS-4a8cd6c0-09d4-48dd-8482-f9564e847002 from datanode DatanodeRegistration(127.0.0.1:45769, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=38151, infoSecurePort=0, ipcPort=45383, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504)
2024-12-10T02:25:20,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43a3d75df02a3774 with lease ID 0x2c1a4f584b5ecf3a: from storage DS-4a8cd6c0-09d4-48dd-8482-f9564e847002 node DatanodeRegistration(127.0.0.1:45769, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=38151, infoSecurePort=0, ipcPort=45383, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:20,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43a3d75df02a3774 with lease ID 0x2c1a4f584b5ecf3a: Processing first storage report for DS-5a6f8ff2-df73-40f0-a845-20db3762e526 from datanode DatanodeRegistration(127.0.0.1:45769, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=38151, infoSecurePort=0, ipcPort=45383, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504)
2024-12-10T02:25:20,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43a3d75df02a3774 with lease ID 0x2c1a4f584b5ecf3a: from storage DS-5a6f8ff2-df73-40f0-a845-20db3762e526 node DatanodeRegistration(127.0.0.1:45769, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=38151, infoSecurePort=0, ipcPort=45383, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:20,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1fc8bed8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-38325-hadoop-hdfs-3_4_1-tests_jar-_-any-17607242953382338383/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:20,046 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:38325}
2024-12-10T02:25:20,046 INFO  [Time-limited test {}] server.Server(415): Started @149105ms
2024-12-10T02:25:20,048 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:20,155 WARN  [Thread-1217 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data3/current/BP-1457422663-172.17.0.2-1733797519504/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:20,155 WARN  [Thread-1218 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data4/current/BP-1457422663-172.17.0.2-1733797519504/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:20,174 WARN  [Thread-1206 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:20,176 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65224b69c6477a5a with lease ID 0x2c1a4f584b5ecf3b: Processing first storage report for DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20 from datanode DatanodeRegistration(127.0.0.1:46553, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=41141, infoSecurePort=0, ipcPort=44985, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504)
2024-12-10T02:25:20,177 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65224b69c6477a5a with lease ID 0x2c1a4f584b5ecf3b: from storage DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20 node DatanodeRegistration(127.0.0.1:46553, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=41141, infoSecurePort=0, ipcPort=44985, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T02:25:20,177 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x65224b69c6477a5a with lease ID 0x2c1a4f584b5ecf3b: Processing first storage report for DS-12ed726f-da44-4c78-8e3b-c7b293effd6e from datanode DatanodeRegistration(127.0.0.1:46553, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=41141, infoSecurePort=0, ipcPort=44985, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504)
2024-12-10T02:25:20,177 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x65224b69c6477a5a with lease ID 0x2c1a4f584b5ecf3b: from storage DS-12ed726f-da44-4c78-8e3b-c7b293effd6e node DatanodeRegistration(127.0.0.1:46553, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=41141, infoSecurePort=0, ipcPort=44985, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:20,273 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8
2024-12-10T02:25:20,276 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/zookeeper_0, clientPort=62310, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:25:20,277 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62310
2024-12-10T02:25:20,277 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,279 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:25:20,287 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:25:20,289 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6 with version=8
2024-12-10T02:25:20,289 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:25:20,291 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:25:20,292 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44767
2024-12-10T02:25:20,293 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44767 connecting to ZooKeeper ensemble=127.0.0.1:62310
2024-12-10T02:25:20,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447670x0, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:25:20,301 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44767-0x1019a308e220000 connected
2024-12-10T02:25:20,317 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,319 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,321 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:20,321 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6, hbase.cluster.distributed=false
2024-12-10T02:25:20,322 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:25:20,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44767
2024-12-10T02:25:20,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44767
2024-12-10T02:25:20,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44767
2024-12-10T02:25:20,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44767
2024-12-10T02:25:20,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44767
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:25:20,340 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:25:20,341 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35203
2024-12-10T02:25:20,342 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35203 connecting to ZooKeeper ensemble=127.0.0.1:62310
2024-12-10T02:25:20,343 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,345 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352030x0, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:25:20,349 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35203-0x1019a308e220001 connected
2024-12-10T02:25:20,349 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:20,349 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:25:20,350 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:25:20,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:25:20,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:25:20,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35203
2024-12-10T02:25:20,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35203
2024-12-10T02:25:20,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35203
2024-12-10T02:25:20,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35203
2024-12-10T02:25:20,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35203
2024-12-10T02:25:20,365 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:44767
2024-12-10T02:25:20,365 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:20,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:20,367 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:25:20,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,369 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:25:20,369 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,44767,1733797520290 from backup master directory
2024-12-10T02:25:20,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:20,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:20,371 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:25:20,371 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,376 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/hbase.id] with ID: 13e9519a-bede-4ad8-b4b7-f071929811bf
2024-12-10T02:25:20,376 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/.tmp/hbase.id
2024-12-10T02:25:20,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:25:20,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:25:20,382 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/.tmp/hbase.id]:[hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/hbase.id]
2024-12-10T02:25:20,393 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:20,393 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:25:20,395 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-10T02:25:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,405 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:25:20,406 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:25:20,406 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:25:20,407 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:25:20,408 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:20,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:25:20,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:25:20,418 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store
2024-12-10T02:25:20,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:25:20,425 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:25:20,426 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:20,426 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:20,426 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797520426Disabling compacts and flushes for region at 1733797520426Disabling writes for close at 1733797520426Writing region close event to WAL at 1733797520426Closed at 1733797520426
2024-12-10T02:25:20,427 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/.initializing
2024-12-10T02:25:20,427 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C44767%2C1733797520290, suffix=, logDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290, archiveDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/oldWALs, maxLogs=10
2024-12-10T02:25:20,430 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C44767%2C1733797520290.1733797520430
2024-12-10T02:25:20,435 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430
2024-12-10T02:25:20,435 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38151:38151),(127.0.0.1/127.0.0.1:41141:41141)]
2024-12-10T02:25:20,436 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:20,436 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:20,436 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,436 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,438 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:20,439 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:25:20,439 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,439 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:20,440 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:20,441 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:25:20,441 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,441 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:20,441 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:20,442 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:25:20,442 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,443 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:20,443 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:20,444 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:25:20,444 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,445 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:20,445 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,446 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,446 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,448 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,448 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,449 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:25:20,450 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:20,453 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:20,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=756180, jitterRate=-0.03846845030784607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:25:20,454 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797520436Initializing all the Stores at 1733797520437 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797520437Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797520437Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797520437Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797520437Cleaning up temporary data from old regions at 1733797520448 (+11 ms)Region opened successfully at 1733797520454 (+6 ms)
2024-12-10T02:25:20,454 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:25:20,457 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc58a03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:25:20,458 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:25:20,458 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:25:20,458 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:25:20,458 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:25:20,459 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:25:20,459 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:25:20,459 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:25:20,461 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:25:20,462 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:25:20,463 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:25:20,464 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:25:20,464 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:25:20,466 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:25:20,466 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:25:20,467 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:25:20,471 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:25:20,472 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:25:20,474 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:25:20,476 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:25:20,477 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:25:20,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:20,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:20,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,480 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,44767,1733797520290, sessionid=0x1019a308e220000, setting cluster-up flag (Was=false)
2024-12-10T02:25:20,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,491 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:25:20,492 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,507 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:25:20,508 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,44767,1733797520290
2024-12-10T02:25:20,510 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:25:20,512 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:20,512 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:25:20,512 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:25:20,513 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,44767,1733797520290
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:25:20,514 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:20,514 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:20,514 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:20,515 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:20,515 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:25:20,515 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,515 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:25:20,515 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,518 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:20,518 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:25:20,520 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,520 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:25:20,520 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797550520
2024-12-10T02:25:20,520 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:25:20,521 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,524 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:25:20,524 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:25:20,525 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:25:20,525 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:25:20,525 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:25:20,525 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797520525,5,FailOnTimeoutGroup]
2024-12-10T02:25:20,525 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797520525,5,FailOnTimeoutGroup]
2024-12-10T02:25:20,525 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,526 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:25:20,526 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,526 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,531 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:25:20,532 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:25:20,533 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:25:20,533 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6
2024-12-10T02:25:20,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:25:20,543 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:25:20,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:20,546 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:25:20,547 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:25:20,548 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,548 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:20,548 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:25:20,550 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:25:20,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,551 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:20,551 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:25:20,552 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:25:20,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,553 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:20,553 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:25:20,555 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:25:20,555 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:20,555 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(746): ClusterId : 13e9519a-bede-4ad8-b4b7-f071929811bf
2024-12-10T02:25:20,555 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:25:20,555 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:20,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:25:20,556 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740
2024-12-10T02:25:20,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740
2024-12-10T02:25:20,558 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:25:20,558 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:25:20,558 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:25:20,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:25:20,559 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:25:20,560 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:25:20,561 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:25:20,561 DEBUG [RS:0;d9f49988d155:35203 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aab0793, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:25:20,563 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:20,563 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854153, jitterRate=0.08611263334751129}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:25:20,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797520544Initializing all the Stores at 1733797520545 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797520545Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797520546 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797520546Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797520546Cleaning up temporary data from old regions at 1733797520559 (+13 ms)Region opened successfully at 1733797520564 (+5 ms)
2024-12-10T02:25:20,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:25:20,564 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:25:20,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:25:20,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:25:20,564 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:25:20,565 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:20,565 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797520564Disabling compacts and flushes for region at 1733797520564Disabling writes for close at 1733797520564Writing region close event to WAL at 1733797520565 (+1 ms)Closed at 1733797520565
2024-12-10T02:25:20,566 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:20,566 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:25:20,566 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:25:20,568 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:25:20,569 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:25:20,578 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:35203
2024-12-10T02:25:20,578 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:25:20,578 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:25:20,578 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:25:20,579 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,44767,1733797520290 with port=35203, startcode=1733797520339
2024-12-10T02:25:20,580 DEBUG [RS:0;d9f49988d155:35203 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:25:20,582 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45937, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:25:20,583 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44767 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,583 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44767 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,584 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6
2024-12-10T02:25:20,585 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33395
2024-12-10T02:25:20,585 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:25:20,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:25:20,588 DEBUG [RS:0;d9f49988d155:35203 {}] zookeeper.ZKUtil(111): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,588 WARN  [RS:0;d9f49988d155:35203 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:25:20,588 INFO  [RS:0;d9f49988d155:35203 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:20,588 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,588 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,35203,1733797520339]
2024-12-10T02:25:20,592 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:25:20,595 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:25:20,596 INFO  [RS:0;d9f49988d155:35203 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:25:20,596 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,597 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:25:20,598 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:25:20,598 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:25:20,598 DEBUG [RS:0;d9f49988d155:35203 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:25:20,599 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,599 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,599 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,600 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,600 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,600 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,35203,1733797520339-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:25:20,617 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:25:20,617 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,35203,1733797520339-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,617 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,617 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.Replication(171): d9f49988d155,35203,1733797520339 started
2024-12-10T02:25:20,632 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:20,632 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,35203,1733797520339, RpcServer on d9f49988d155/172.17.0.2:35203, sessionid=0x1019a308e220001
2024-12-10T02:25:20,633 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:25:20,633 DEBUG [RS:0;d9f49988d155:35203 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,633 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,35203,1733797520339'
2024-12-10T02:25:20,633 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:25:20,633 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,35203,1733797520339'
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:25:20,634 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:25:20,635 DEBUG [RS:0;d9f49988d155:35203 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:25:20,635 INFO  [RS:0;d9f49988d155:35203 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:25:20,635 INFO  [RS:0;d9f49988d155:35203 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:25:20,719 WARN  [d9f49988d155:44767 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:25:20,737 INFO  [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C35203%2C1733797520339, suffix=, logDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339, archiveDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs, maxLogs=32
2024-12-10T02:25:20,738 INFO  [RS:0;d9f49988d155:35203 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:20,744 INFO  [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:20,749 DEBUG [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38151:38151),(127.0.0.1/127.0.0.1:41141:41141)]
2024-12-10T02:25:20,969 DEBUG [d9f49988d155:44767 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:25:20,970 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,35203,1733797520339
2024-12-10T02:25:20,971 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,35203,1733797520339, state=OPENING
2024-12-10T02:25:20,973 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:25:20,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:20,975 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:25:20,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:20,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:20,975 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,35203,1733797520339}]
2024-12-10T02:25:20,990 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:21,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:21,128 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:25:21,130 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38179, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:25:21,134 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:25:21,134 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:21,136 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C35203%2C1733797520339.meta, suffix=.meta, logDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339, archiveDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs, maxLogs=32
2024-12-10T02:25:21,136 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta
2024-12-10T02:25:21,141 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta
2024-12-10T02:25:21,142 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41141:41141),(127.0.0.1/127.0.0.1:38151:38151)]
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:25:21,143 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:25:21,143 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:25:21,145 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:25:21,145 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:25:21,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:21,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:25:21,147 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:25:21,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,147 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:21,147 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:25:21,148 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:25:21,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:21,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:25:21,149 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:25:21,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,149 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:21,149 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:25:21,150 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740
2024-12-10T02:25:21,151 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740
2024-12-10T02:25:21,152 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:25:21,152 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:25:21,153 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:25:21,154 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:25:21,155 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=794128, jitterRate=0.009786352515220642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:25:21,155 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:25:21,156 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797521143Writing region info on filesystem at 1733797521144 (+1 ms)Initializing all the Stores at 1733797521144Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797521144Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797521144Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797521144Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797521144Cleaning up temporary data from old regions at 1733797521152 (+8 ms)Running coprocessor post-open hooks at 1733797521155 (+3 ms)Region opened successfully at 1733797521156 (+1 ms)
2024-12-10T02:25:21,157 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797521128
2024-12-10T02:25:21,159 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:25:21,159 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:25:21,160 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,35203,1733797520339
2024-12-10T02:25:21,161 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,35203,1733797520339, state=OPEN
2024-12-10T02:25:21,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:25:21,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:25:21,172 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,35203,1733797520339
2024-12-10T02:25:21,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:21,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:21,175 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:25:21,175 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,35203,1733797520339 in 197 msec
2024-12-10T02:25:21,178 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:25:21,178 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec
2024-12-10T02:25:21,179 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:21,179 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:25:21,180 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:25:21,180 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,35203,1733797520339, seqNum=-1]
2024-12-10T02:25:21,180 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:25:21,182 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57969, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:25:21,187 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 675 msec
2024-12-10T02:25:21,187 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797521187, completionTime=-1
2024-12-10T02:25:21,187 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:25:21,187 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:25:21,188 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797581189
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797641189
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:44767, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,191 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.822sec
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:25:21,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:25:21,195 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:25:21,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:25:21,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,44767,1733797520290-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:21,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67332887, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:21,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,44767,-1 for getting cluster id
2024-12-10T02:25:21,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:25:21,257 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '13e9519a-bede-4ad8-b4b7-f071929811bf'
2024-12-10T02:25:21,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:25:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "13e9519a-bede-4ad8-b4b7-f071929811bf"

2024-12-10T02:25:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b7b18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,44767,-1]
2024-12-10T02:25:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:25:21,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:21,260 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:25:21,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e9dc43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:21,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:25:21,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,35203,1733797520339, seqNum=-1]
2024-12-10T02:25:21,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:25:21,264 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:25:21,266 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,44767,1733797520290
2024-12-10T02:25:21,266 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:21,269 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:25:21,269 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-12-10T02:25:21,269 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-12-10T02:25:21,269 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T02:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is d9f49988d155,44767,1733797520290
2024-12-10T02:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@74865c2b
2024-12-10T02:25:21,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T02:25:21,271 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T02:25:21,272 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T02:25:21,272 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T02:25:21,273 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:25:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T02:25:21,275 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T02:25:21,275 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,275 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-12-10T02:25:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:25:21,276 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T02:25:21,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741835_1011 (size=395)
2024-12-10T02:25:21,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741835_1011 (size=395)
2024-12-10T02:25:21,285 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0665643476290461445412a196f9b57b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6
2024-12-10T02:25:21,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46553 is added to blk_1073741836_1012 (size=78)
2024-12-10T02:25:21,291 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45769 is added to blk_1073741836_1012 (size=78)
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 0665643476290461445412a196f9b57b, disabling compactions & flushes
2024-12-10T02:25:21,292 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b. after waiting 0 ms
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,292 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,292 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0665643476290461445412a196f9b57b:
Waiting for close lock at 1733797521292Disabling compacts and flushes for region at 1733797521292Disabling writes for close at 1733797521292Writing region close event to WAL at 1733797521292Closed at 1733797521292
2024-12-10T02:25:21,293 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T02:25:21,294 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1733797521293"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797521293"}]},"ts":"1733797521293"}
2024-12-10T02:25:21,296 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T02:25:21,297 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T02:25:21,297 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797521297"}]},"ts":"1733797521297"}
2024-12-10T02:25:21,299 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-12-10T02:25:21,299 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0665643476290461445412a196f9b57b, ASSIGN}]
2024-12-10T02:25:21,301 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0665643476290461445412a196f9b57b, ASSIGN
2024-12-10T02:25:21,302 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0665643476290461445412a196f9b57b, ASSIGN; state=OFFLINE, location=d9f49988d155,35203,1733797520339; forceNewPlan=false, retain=false
2024-12-10T02:25:21,453 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0665643476290461445412a196f9b57b, regionState=OPENING, regionLocation=d9f49988d155,35203,1733797520339
2024-12-10T02:25:21,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0665643476290461445412a196f9b57b, ASSIGN because future has completed
2024-12-10T02:25:21,456 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0665643476290461445412a196f9b57b, server=d9f49988d155,35203,1733797520339}]
2024-12-10T02:25:21,612 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,612 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0665643476290461445412a196f9b57b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:21,612 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,612 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:21,612 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,612 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,614 INFO  [StoreOpener-0665643476290461445412a196f9b57b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0665643476290461445412a196f9b57b 
2024-12-10T02:25:21,615 INFO  [StoreOpener-0665643476290461445412a196f9b57b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0665643476290461445412a196f9b57b columnFamilyName info
2024-12-10T02:25:21,615 DEBUG [StoreOpener-0665643476290461445412a196f9b57b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:21,615 INFO  [StoreOpener-0665643476290461445412a196f9b57b-1 {}] regionserver.HStore(327): Store=0665643476290461445412a196f9b57b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:21,615 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,616 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b
2024-12-10T02:25:21,616 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b
2024-12-10T02:25:21,617 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,617 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,618 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,620 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:21,621 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0665643476290461445412a196f9b57b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713671, jitterRate=-0.09252163767814636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:25:21,621 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0665643476290461445412a196f9b57b
2024-12-10T02:25:21,621 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0665643476290461445412a196f9b57b:
Running coprocessor pre-open hook at 1733797521612Writing region info on filesystem at 1733797521613 (+1 ms)Initializing all the Stores at 1733797521613Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797521613Cleaning up temporary data from old regions at 1733797521617 (+4 ms)Running coprocessor post-open hooks at 1733797521621 (+4 ms)Region opened successfully at 1733797521621
2024-12-10T02:25:21,622 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b., pid=6, masterSystemTime=1733797521608
2024-12-10T02:25:21,625 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,625 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:21,626 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0665643476290461445412a196f9b57b, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,35203,1733797520339
2024-12-10T02:25:21,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0665643476290461445412a196f9b57b, server=d9f49988d155,35203,1733797520339 because future has completed
2024-12-10T02:25:21,631 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T02:25:21,632 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0665643476290461445412a196f9b57b, server=d9f49988d155,35203,1733797520339 in 173 msec
2024-12-10T02:25:21,634 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T02:25:21,634 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=0665643476290461445412a196f9b57b, ASSIGN in 333 msec
2024-12-10T02:25:21,635 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T02:25:21,635 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797521635"}]},"ts":"1733797521635"}
2024-12-10T02:25:21,637 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-12-10T02:25:21,638 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T02:25:21,640 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 365 msec
2024-12-10T02:25:21,991 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:22,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:22,992 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:23,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:23,993 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:24,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:24,993 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:25,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:25,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:26,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:26,645 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:25:26,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,665 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,665 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,666 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,670 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,672 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:26,678 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:25:26,678 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T02:25:26,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T02:25:26,679 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-12-10T02:25:26,680 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:26,680 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T02:25:26,680 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:25:26,681 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-12-10T02:25:26,994 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:27,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:27,995 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:28,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:28,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:29,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:29,996 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:30,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:30,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:31,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:31,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44767 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:25:31,344 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-12-10T02:25:31,345 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-12-10T02:25:31,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T02:25:31,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b., hostname=d9f49988d155,35203,1733797520339, seqNum=2]
2024-12-10T02:25:31,997 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:32,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:32,998 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:33,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:33,354 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:33,355 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,355 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,356 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK], DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]) is bad.
2024-12-10T02:25:33,356 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK], DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]) is bad.
2024-12-10T02:25:33,355 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,356 WARN  [PacketResponder: BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46553] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,356 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK], DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46553,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]) is bad.
2024-12-10T02:25:33,356 WARN  [PacketResponder: BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46553] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:46010 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46010 dst: /127.0.0.1:45769
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:47002 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46553:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:47002 dst: /127.0.0.1:46553
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,356 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:46988 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46553:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46988 dst: /127.0.0.1:46553
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,357 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:46020 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46020 dst: /127.0.0.1:45769
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,357 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1063273559_22 at /127.0.0.1:37982 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37982 dst: /127.0.0.1:45769
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,357 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1063273559_22 at /127.0.0.1:57622 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46553:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:57622 dst: /127.0.0.1:46553
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,358 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1fc8bed8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:33,359 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44bd8b73{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:33,359 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:33,359 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a9fc515{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:33,359 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38cf0a15{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:33,360 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:33,360 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:33,361 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid 7edefdd4-15d7-44d6-870a-075e2b64a7be) service to localhost/127.0.0.1:33395
2024-12-10T02:25:33,361 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:33,361 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data3/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:33,362 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data4/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:33,362 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:33,370 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:33,373 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:33,374 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:33,374 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:33,374 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:33,375 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e01093b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:33,375 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2b5f8885{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:33,490 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2d648826{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-46741-hadoop-hdfs-3_4_1-tests_jar-_-any-10930073019130686893/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:33,491 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a8d3987{HTTP/1.1, (http/1.1)}{localhost:46741}
2024-12-10T02:25:33,491 INFO  [Time-limited test {}] server.Server(415): Started @162549ms
2024-12-10T02:25:33,492 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:33,510 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,510 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,510 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:33,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:37240 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37240 dst: /127.0.0.1:45769
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:37238 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37238 dst: /127.0.0.1:45769
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,512 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1063273559_22 at /127.0.0.1:37224 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45769:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37224 dst: /127.0.0.1:45769
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:33,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21865735{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:33,518 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54562211{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:33,518 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:33,518 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fefca8b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:33,519 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc14744{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:33,520 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:33,520 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid eab0cac4-6345-4706-b8fe-8858c6dc8917) service to localhost/127.0.0.1:33395
2024-12-10T02:25:33,520 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:33,520 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:33,524 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data1/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:33,524 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data2/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:33,524 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:33,532 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:33,535 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:33,535 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:33,535 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:33,536 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:33,536 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ea055e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:33,536 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@343c572e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:33,591 WARN  [Thread-1341 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:33,593 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f439d44d15a7b3 with lease ID 0x2c1a4f584b5ecf3c: from storage DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20 node DatanodeRegistration(127.0.0.1:42053, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=43071, infoSecurePort=0, ipcPort=33427, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:33,593 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2f439d44d15a7b3 with lease ID 0x2c1a4f584b5ecf3c: from storage DS-12ed726f-da44-4c78-8e3b-c7b293effd6e node DatanodeRegistration(127.0.0.1:42053, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=43071, infoSecurePort=0, ipcPort=33427, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:33,652 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fd8f940{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-46625-hadoop-hdfs-3_4_1-tests_jar-_-any-1976931330613616300/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:33,652 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cbeb460{HTTP/1.1, (http/1.1)}{localhost:46625}
2024-12-10T02:25:33,652 INFO  [Time-limited test {}] server.Server(415): Started @162711ms
2024-12-10T02:25:33,654 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:33,782 WARN  [Thread-1372 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:33,785 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36d032b1d7162a39 with lease ID 0x2c1a4f584b5ecf3d: from storage DS-4a8cd6c0-09d4-48dd-8482-f9564e847002 node DatanodeRegistration(127.0.0.1:42843, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=37569, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:33,785 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36d032b1d7162a39 with lease ID 0x2c1a4f584b5ecf3d: from storage DS-5a6f8ff2-df73-40f0-a845-20db3762e526 node DatanodeRegistration(127.0.0.1:42843, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=37569, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:33,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:34,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:34,672 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-12-10T02:25:34,675 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-12-10T02:25:34,676 ERROR [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:34,676 WARN  [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:34,676 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C35203%2C1733797520339:(num 1733797520738) roll requested
2024-12-10T02:25:34,677 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:34,682 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 newFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:34,682 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:34,682 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:34,682 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:34,683 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:34,683 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:34,683 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:34,683 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:34,683 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:34,683 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:34,684 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43071:43071),(127.0.0.1/127.0.0.1:37569:37569)]
2024-12-10T02:25:34,684 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 is not closed yet, will try archiving it next time
2024-12-10T02:25:34,684 WARN  [IPC Server handler 3 on default port 33395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014
2024-12-10T02:25:34,684 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 after 1ms
2024-12-10T02:25:34,999 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:35,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:36,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:36,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:36,687 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-12-10T02:25:37,000 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:37,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:38,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:38,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:38,593 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T02:25:38,685 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 after 4002ms
2024-12-10T02:25:38,690 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:42843,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:38,690 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42053,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK], DatanodeInfoWithStorage[127.0.0.1:42843,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42843,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]) is bad.
2024-12-10T02:25:38,690 WARN  [PacketResponder: BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42843] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:38,690 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:50584 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:42053:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50584 dst: /127.0.0.1:42053
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:38,691 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:53334 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:42843:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:53334 dst: /127.0.0.1:42843
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:38,692 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fd8f940{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:38,692 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cbeb460{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:38,692 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:38,692 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@343c572e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:38,693 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ea055e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:38,694 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:38,694 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:38,694 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:38,694 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid eab0cac4-6345-4706-b8fe-8858c6dc8917) service to localhost/127.0.0.1:33395
2024-12-10T02:25:38,695 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data1/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:38,695 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data2/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:38,695 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:38,705 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:38,709 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:38,710 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:38,710 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:38,710 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:38,710 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45c55ac5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:38,710 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a382d25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:38,824 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@362ea065{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-35337-hadoop-hdfs-3_4_1-tests_jar-_-any-10069765174433976613/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:38,824 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78cc0925{HTTP/1.1, (http/1.1)}{localhost:35337}
2024-12-10T02:25:38,824 INFO  [Time-limited test {}] server.Server(415): Started @167883ms
2024-12-10T02:25:38,826 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:38,849 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:38,849 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-557029929_22 at /127.0.0.1:50606 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:42053:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:50606 dst: /127.0.0.1:42053
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:38,852 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2d648826{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:38,852 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a8d3987{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:38,853 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:38,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2b5f8885{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:38,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e01093b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:38,854 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:38,854 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid 7edefdd4-15d7-44d6-870a-075e2b64a7be) service to localhost/127.0.0.1:33395
2024-12-10T02:25:38,854 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:38,854 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:38,855 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data3/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:38,855 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data4/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:38,855 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:38,867 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:38,871 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:38,871 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:38,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:38,872 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:38,872 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39c5f69b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:38,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5091fc79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:38,925 WARN  [Thread-1415 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1086fc47e1b19f53 with lease ID 0x2c1a4f584b5ecf3e: from storage DS-4a8cd6c0-09d4-48dd-8482-f9564e847002 node DatanodeRegistration(127.0.0.1:41993, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=34193, infoSecurePort=0, ipcPort=45495, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:38,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1086fc47e1b19f53 with lease ID 0x2c1a4f584b5ecf3e: from storage DS-5a6f8ff2-df73-40f0-a845-20db3762e526 node DatanodeRegistration(127.0.0.1:41993, datanodeUuid=eab0cac4-6345-4706-b8fe-8858c6dc8917, infoPort=34193, infoSecurePort=0, ipcPort=45495, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:38,990 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@262fed34{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/java.io.tmpdir/jetty-localhost-45247-hadoop-hdfs-3_4_1-tests_jar-_-any-2581446130057985403/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:38,990 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f7ae33f{HTTP/1.1, (http/1.1)}{localhost:45247}
2024-12-10T02:25:38,990 INFO  [Time-limited test {}] server.Server(415): Started @168049ms
2024-12-10T02:25:38,992 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:39,001 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:39,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:39,082 WARN  [Thread-1446 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:39,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb74ed639dd04b4ec with lease ID 0x2c1a4f584b5ecf3f: from storage DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20 node DatanodeRegistration(127.0.0.1:35443, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=37437, infoSecurePort=0, ipcPort=34831, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:39,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb74ed639dd04b4ec with lease ID 0x2c1a4f584b5ecf3f: from storage DS-12ed726f-da44-4c78-8e3b-c7b293effd6e node DatanodeRegistration(127.0.0.1:35443, datanodeUuid=7edefdd4-15d7-44d6-870a-075e2b64a7be, infoPort=37437, infoSecurePort=0, ipcPort=34831, storageInfo=lv=-57;cid=testClusterID;nsid=680551409;c=1733797519504), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:40,002 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:40,010 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-12-10T02:25:40,012 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-12-10T02:25:40,013 ERROR [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42053,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:40,013 WARN  [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42053,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:40,013 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C35203%2C1733797520339:(num 1733797534677) roll requested
2024-12-10T02:25:40,013 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:40,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:40,019 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 newFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:40,019 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:40,019 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:40,019 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:40,019 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:40,019 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:40,019 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:40,020 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42053,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:40,020 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42053,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:40,020 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:40,020 WARN  [IPC Server handler 1 on default port 33395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-12-10T02:25:40,020 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 after 0ms
2024-12-10T02:25:40,023 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34193:34193),(127.0.0.1/127.0.0.1:37437:37437)]
2024-12-10T02:25:40,023 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 is not closed yet, will try archiving it next time
2024-12-10T02:25:40,927 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T02:25:41,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:41,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:42,003 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:42,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:42,024 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:42,030 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013 newFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:42,030 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:42,030 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:42,030 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:42,030 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:42,030 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:42,031 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:42,031 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34193:34193),(127.0.0.1/127.0.0.1:37437:37437)]
2024-12-10T02:25:42,032 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 is not closed yet, will try archiving it next time
2024-12-10T02:25:42,032 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013 is not closed yet, will try archiving it next time
2024-12-10T02:25:42,032 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:42,032 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:42,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741838_1019 (size=1264)
2024-12-10T02:25:42,033 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741838_1019 (size=1264)
2024-12-10T02:25:42,033 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738 after 1ms
2024-12-10T02:25:42,033 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:42,033 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 is not closed yet, will try archiving it next time
2024-12-10T02:25:42,041 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1733797521622/Put/vlen=218/seqid=0]
2024-12-10T02:25:42,041 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1733797531352/Put/vlen=1045/seqid=0]
2024-12-10T02:25:42,041 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797520738
2024-12-10T02:25:42,041 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:42,041 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:42,042 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 after 1ms
2024-12-10T02:25:42,042 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:42,045 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1733797534676/Put/vlen=1045/seqid=0]
2024-12-10T02:25:42,045 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1733797536688/Put/vlen=1045/seqid=0]
2024-12-10T02:25:42,045 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677
2024-12-10T02:25:42,045 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:42,045 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:42,046 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013 after 0ms
2024-12-10T02:25:42,046 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797540013
2024-12-10T02:25:42,048 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1733797540013/Put/vlen=1045/seqid=0]
2024-12-10T02:25:42,048 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:42,048 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:42,049 WARN  [IPC Server handler 0 on default port 33395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-12-10T02:25:42,049 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 after 1ms
2024-12-10T02:25:42,928 WARN  [ResponseProcessor for block BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:42,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1063273559_22 at /127.0.0.1:34056 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41993:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34056 dst: /127.0.0.1:41993
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41993 remote=/127.0.0.1:34056]. Total timeout mills is 60000, 59102 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:42,928 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1063273559_22 at /127.0.0.1:38582 [Receiving block BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:35443:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:38582 dst: /127.0.0.1:35443
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:25:42,928 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 block BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41993,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK], DatanodeInfoWithStorage[127.0.0.1:35443,DS-bd9ce4ba-7581-430c-ae35-15e03f6e2e20,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41993,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]) is bad.
2024-12-10T02:25:42,929 WARN  [DataStreamer for file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 block BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:42,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741839_1022 (size=85)
2024-12-10T02:25:42,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741839_1022 (size=85)
2024-12-10T02:25:43,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:43,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:44,004 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:44,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:44,022 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797534677 after 4002ms
2024-12-10T02:25:45,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:45,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:46,005 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:46,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:46,050 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 after 4002ms
2024-12-10T02:25:46,050 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:46,053 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:46,054 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 0665643476290461445412a196f9b57b 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-12-10T02:25:46,054 ERROR [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,055 WARN  [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,055 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C35203%2C1733797520339:(num 1733797542024) roll requested
2024-12-10T02:25:46,055 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.1733797546055
2024-12-10T02:25:46,060 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 newFile=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797546055
2024-12-10T02:25:46,060 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,060 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,060 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,060 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,060 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,060 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797546055
2024-12-10T02:25:46,061 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,061 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1457422663-172.17.0.2-1733797519504:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,061 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:46,062 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37437:37437),(127.0.0.1/127.0.0.1:34193:34193)]
2024-12-10T02:25:46,062 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 after 1ms
2024-12-10T02:25:46,062 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 is not closed yet, will try archiving it next time
2024-12-10T02:25:46,062 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.1733797542024 to hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs/d9f49988d155%2C35203%2C1733797520339.1733797542024
2024-12-10T02:25:46,077 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/.tmp/info/8ed162e3808f4efda0f4cd65cd46616d is 1080, key is row1002/info:/1733797531352/Put/seqid=0
2024-12-10T02:25:46,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741841_1024 (size=9270)
2024-12-10T02:25:46,082 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741841_1024 (size=9270)
2024-12-10T02:25:46,082 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/.tmp/info/8ed162e3808f4efda0f4cd65cd46616d
2024-12-10T02:25:46,088 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/.tmp/info/8ed162e3808f4efda0f4cd65cd46616d as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/info/8ed162e3808f4efda0f4cd65cd46616d
2024-12-10T02:25:46,093 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/info/8ed162e3808f4efda0f4cd65cd46616d, entries=4, sequenceid=8, filesize=9.1 K
2024-12-10T02:25:46,094 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 0665643476290461445412a196f9b57b in 40ms, sequenceid=8, compaction requested=false
2024-12-10T02:25:46,094 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0665643476290461445412a196f9b57b:

2024-12-10T02:25:46,095 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-12-10T02:25:46,095 ERROR [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,095 WARN  [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6-prefix:d9f49988d155,35203,1733797520339.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,095 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C35203%2C1733797520339.meta:.meta(num 1733797521136) roll requested
2024-12-10T02:25:46,095 INFO  [regionserver/d9f49988d155:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C35203%2C1733797520339.meta.1733797546095.meta
2024-12-10T02:25:46,100 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,100 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,100 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,100 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,100 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,100 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797546095.meta
2024-12-10T02:25:46,101 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,101 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:46,101 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta
2024-12-10T02:25:46,101 WARN  [IPC Server handler 3 on default port 33395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1015
2024-12-10T02:25:46,102 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta after 1ms
2024-12-10T02:25:46,104 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37437:37437),(127.0.0.1/127.0.0.1:34193:34193)]
2024-12-10T02:25:46,104 DEBUG [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta is not closed yet, will try archiving it next time
2024-12-10T02:25:46,119 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/info/f2d89873e1604917aa06776804aa0465 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b./info:regioninfo/1733797521625/Put/seqid=0
2024-12-10T02:25:46,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741843_1027 (size=7125)
2024-12-10T02:25:46,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741843_1027 (size=7125)
2024-12-10T02:25:46,124 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/info/f2d89873e1604917aa06776804aa0465
2024-12-10T02:25:46,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/ns/395af6c8eab4499581e84787cc5e743d is 43, key is default/ns:d/1733797521182/Put/seqid=0
2024-12-10T02:25:46,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741844_1028 (size=5153)
2024-12-10T02:25:46,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741844_1028 (size=5153)
2024-12-10T02:25:46,148 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/ns/395af6c8eab4499581e84787cc5e743d
2024-12-10T02:25:46,174 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/table/1daf9c6db10549f59ffba0f33e5563e0 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1733797521635/Put/seqid=0
2024-12-10T02:25:46,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741845_1029 (size=5438)
2024-12-10T02:25:46,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741845_1029 (size=5438)
2024-12-10T02:25:46,181 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/table/1daf9c6db10549f59ffba0f33e5563e0
2024-12-10T02:25:46,186 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/info/f2d89873e1604917aa06776804aa0465 as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/info/f2d89873e1604917aa06776804aa0465
2024-12-10T02:25:46,191 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/info/f2d89873e1604917aa06776804aa0465, entries=10, sequenceid=11, filesize=7.0 K
2024-12-10T02:25:46,192 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/ns/395af6c8eab4499581e84787cc5e743d as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/ns/395af6c8eab4499581e84787cc5e743d
2024-12-10T02:25:46,197 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/ns/395af6c8eab4499581e84787cc5e743d, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T02:25:46,198 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/.tmp/table/1daf9c6db10549f59ffba0f33e5563e0 as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/table/1daf9c6db10549f59ffba0f33e5563e0
2024-12-10T02:25:46,202 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/table/1daf9c6db10549f59ffba0f33e5563e0, entries=2, sequenceid=11, filesize=5.3 K
2024-12-10T02:25:46,203 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=11, compaction requested=false
2024-12-10T02:25:46,203 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T02:25:46,210 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:25:46,210 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:25:46,210 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:46,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:46,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:46,210 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:25:46,211 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:25:46,211 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=79262945, stopped=false
2024-12-10T02:25:46,211 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,44767,1733797520290
2024-12-10T02:25:46,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:46,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:46,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:46,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:46,213 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:25:46,213 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:25:46,213 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:46,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:46,213 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:46,214 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,35203,1733797520339' *****
2024-12-10T02:25:46,214 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:25:46,214 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:46,214 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(3091): Received CLOSE for 0665643476290461445412a196f9b57b
2024-12-10T02:25:46,215 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,35203,1733797520339
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:35203.
2024-12-10T02:25:46,215 DEBUG [RS:0;d9f49988d155:35203 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:25:46,215 DEBUG [RS:0;d9f49988d155:35203 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:46,215 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0665643476290461445412a196f9b57b, disabling compactions & flushes
2024-12-10T02:25:46,215 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:46,215 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:46,216 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:25:46,216 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b. after waiting 0 ms
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:46,216 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:25:46,216 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T02:25:46,216 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1325): Online Regions={0665643476290461445412a196f9b57b=TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b., 1588230740=hbase:meta,,1.1588230740}
2024-12-10T02:25:46,216 DEBUG [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1351): Waiting on 0665643476290461445412a196f9b57b, 1588230740
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:25:46,216 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:25:46,216 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:25:46,225 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T02:25:46,226 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:46,226 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:46,226 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797546216Running coprocessor pre-close hooks at 1733797546216Disabling compacts and flushes for region at 1733797546216Disabling writes for close at 1733797546216Writing region close event to WAL at 1733797546221 (+5 ms)Running coprocessor post-close hooks at 1733797546226 (+5 ms)Closed at 1733797546226
2024-12-10T02:25:46,226 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:46,230 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/data/default/TestLogRolling-testLogRollOnPipelineRestart/0665643476290461445412a196f9b57b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-12-10T02:25:46,230 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:46,231 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0665643476290461445412a196f9b57b:
Waiting for close lock at 1733797546215Running coprocessor pre-close hooks at 1733797546215Disabling compacts and flushes for region at 1733797546215Disabling writes for close at 1733797546216 (+1 ms)Writing region close event to WAL at 1733797546224 (+8 ms)Running coprocessor post-close hooks at 1733797546230 (+6 ms)Closed at 1733797546230
2024-12-10T02:25:46,231 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1733797521272.0665643476290461445412a196f9b57b.
2024-12-10T02:25:46,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:46,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:25:46,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-12-10T02:25:46,416 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,35203,1733797520339; all regions closed.
2024-12-10T02:25:46,417 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,417 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,417 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,417 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,417 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:46,419 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741842_1025 (size=825)
2024-12-10T02:25:46,420 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741842_1025 (size=825)
2024-12-10T02:25:46,600 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-12-10T02:25:46,600 INFO  [regionserver/d9f49988d155:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-12-10T02:25:46,601 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:47,006 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:47,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:48,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:48,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:49,007 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:49,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:50,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:50,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:50,088 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T02:25:50,102 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta after 4001ms
2024-12-10T02:25:50,103 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/WALs/d9f49988d155,35203,1733797520339/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta to hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs/d9f49988d155%2C35203%2C1733797520339.meta.1733797521136.meta
2024-12-10T02:25:50,105 DEBUG [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs
2024-12-10T02:25:50,106 INFO  [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C35203%2C1733797520339.meta:.meta(num 1733797546095)
2024-12-10T02:25:50,106 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,106 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,106 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,106 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,106 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741840_1023 (size=1162)
2024-12-10T02:25:50,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741840_1023 (size=1162)
2024-12-10T02:25:50,113 DEBUG [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs
2024-12-10T02:25:50,113 INFO  [RS:0;d9f49988d155:35203 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C35203%2C1733797520339:(num 1733797546055)
2024-12-10T02:25:50,113 DEBUG [RS:0;d9f49988d155:35203 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:50,113 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:25:50,113 INFO  [RS:0;d9f49988d155:35203 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:25:50,113 INFO  [RS:0;d9f49988d155:35203 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-10T02:25:50,113 INFO  [RS:0;d9f49988d155:35203 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:25:50,114 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:25:50,114 INFO  [RS:0;d9f49988d155:35203 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35203
2024-12-10T02:25:50,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,35203,1733797520339
2024-12-10T02:25:50,116 INFO  [RS:0;d9f49988d155:35203 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:25:50,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:25:50,119 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,35203,1733797520339]
2024-12-10T02:25:50,120 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,35203,1733797520339 already deleted, retry=false
2024-12-10T02:25:50,121 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,35203,1733797520339 expired; onlineServers=0
2024-12-10T02:25:50,121 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,44767,1733797520290' *****
2024-12-10T02:25:50,121 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:25:50,121 INFO  [M:0;d9f49988d155:44767 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:25:50,121 INFO  [M:0;d9f49988d155:44767 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:25:50,121 DEBUG [M:0;d9f49988d155:44767 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:25:50,121 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:25:50,121 DEBUG [M:0;d9f49988d155:44767 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:25:50,121 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797520525 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797520525,5,FailOnTimeoutGroup]
2024-12-10T02:25:50,121 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797520525 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797520525,5,FailOnTimeoutGroup]
2024-12-10T02:25:50,121 INFO  [M:0;d9f49988d155:44767 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:25:50,121 INFO  [M:0;d9f49988d155:44767 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:25:50,121 DEBUG [M:0;d9f49988d155:44767 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:25:50,121 INFO  [M:0;d9f49988d155:44767 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:25:50,122 INFO  [M:0;d9f49988d155:44767 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:25:50,122 INFO  [M:0;d9f49988d155:44767 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:25:50,122 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:25:50,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:25:50,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:50,123 DEBUG [M:0;d9f49988d155:44767 {}] zookeeper.ZKUtil(347): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:25:50,123 WARN  [M:0;d9f49988d155:44767 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:25:50,123 INFO  [M:0;d9f49988d155:44767 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/.lastflushedseqids
2024-12-10T02:25:50,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741846_1030 (size=130)
2024-12-10T02:25:50,129 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741846_1030 (size=130)
2024-12-10T02:25:50,129 INFO  [M:0;d9f49988d155:44767 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:25:50,129 INFO  [M:0;d9f49988d155:44767 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:25:50,129 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:25:50,130 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:50,130 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:50,130 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:25:50,130 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:50,130 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.16 KB heapSize=29.13 KB
2024-12-10T02:25:50,130 ERROR [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData-prefix:d9f49988d155,44767,1733797520290 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:50,130 WARN  [FSHLog-0-hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData-prefix:d9f49988d155,44767,1733797520290 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:50,130 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog d9f49988d155%2C44767%2C1733797520290:(num 1733797520430) roll requested
2024-12-10T02:25:50,131 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C44767%2C1733797520290.1733797550131
2024-12-10T02:25:50,135 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,135 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,135 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,136 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,136 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,136 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 with entries=53, filesize=26.61 KB; new WAL /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797550131
2024-12-10T02:25:50,136 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:50,136 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45769,DS-4a8cd6c0-09d4-48dd-8482-f9564e847002,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-12-10T02:25:50,136 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430
2024-12-10T02:25:50,137 WARN  [IPC Server handler 2 on default port 33395 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013
2024-12-10T02:25:50,137 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 after 1ms
2024-12-10T02:25:50,140 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34193:34193),(127.0.0.1/127.0.0.1:37437:37437)]
2024-12-10T02:25:50,140 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 is not closed yet, will try archiving it next time
2024-12-10T02:25:50,155 DEBUG [M:0;d9f49988d155:44767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/714622592fcf46589310e6c5d8f407e7 is 82, key is hbase:meta,,1/info:regioninfo/1733797521160/Put/seqid=0
2024-12-10T02:25:50,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741848_1033 (size=5672)
2024-12-10T02:25:50,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741848_1033 (size=5672)
2024-12-10T02:25:50,161 INFO  [M:0;d9f49988d155:44767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/714622592fcf46589310e6c5d8f407e7
2024-12-10T02:25:50,181 DEBUG [M:0;d9f49988d155:44767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85140d89c04d4e2fb75df994d2255d09 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733797521639/Put/seqid=0
2024-12-10T02:25:50,185 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741849_1034 (size=6117)
2024-12-10T02:25:50,186 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741849_1034 (size=6117)
2024-12-10T02:25:50,186 INFO  [M:0;d9f49988d155:44767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.56 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85140d89c04d4e2fb75df994d2255d09
2024-12-10T02:25:50,209 DEBUG [M:0;d9f49988d155:44767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e935ea56cf04cd287f0735ec0f2d85e is 69, key is d9f49988d155,35203,1733797520339/rs:state/1733797520583/Put/seqid=0
2024-12-10T02:25:50,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741850_1035 (size=5156)
2024-12-10T02:25:50,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741850_1035 (size=5156)
2024-12-10T02:25:50,214 INFO  [M:0;d9f49988d155:44767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e935ea56cf04cd287f0735ec0f2d85e
2024-12-10T02:25:50,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:50,219 INFO  [RS:0;d9f49988d155:35203 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:25:50,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35203-0x1019a308e220001, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:50,219 INFO  [RS:0;d9f49988d155:35203 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,35203,1733797520339; zookeeper connection closed.
2024-12-10T02:25:50,220 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@29880e9f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@29880e9f
2024-12-10T02:25:50,220 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:25:50,232 DEBUG [M:0;d9f49988d155:44767 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ab3a9a22babc4828b51127e9494ae408 is 52, key is load_balancer_on/state:d/1733797521268/Put/seqid=0
2024-12-10T02:25:50,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741851_1036 (size=5056)
2024-12-10T02:25:50,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741851_1036 (size=5056)
2024-12-10T02:25:50,237 INFO  [M:0;d9f49988d155:44767 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ab3a9a22babc4828b51127e9494ae408
2024-12-10T02:25:50,242 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/714622592fcf46589310e6c5d8f407e7 as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/714622592fcf46589310e6c5d8f407e7
2024-12-10T02:25:50,246 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/714622592fcf46589310e6c5d8f407e7, entries=8, sequenceid=56, filesize=5.5 K
2024-12-10T02:25:50,247 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85140d89c04d4e2fb75df994d2255d09 as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/85140d89c04d4e2fb75df994d2255d09
2024-12-10T02:25:50,253 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/85140d89c04d4e2fb75df994d2255d09, entries=6, sequenceid=56, filesize=6.0 K
2024-12-10T02:25:50,254 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e935ea56cf04cd287f0735ec0f2d85e as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e935ea56cf04cd287f0735ec0f2d85e
2024-12-10T02:25:50,259 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e935ea56cf04cd287f0735ec0f2d85e, entries=1, sequenceid=56, filesize=5.0 K
2024-12-10T02:25:50,260 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ab3a9a22babc4828b51127e9494ae408 as hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ab3a9a22babc4828b51127e9494ae408
2024-12-10T02:25:50,265 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ab3a9a22babc4828b51127e9494ae408, entries=1, sequenceid=56, filesize=4.9 K
2024-12-10T02:25:50,266 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false
2024-12-10T02:25:50,268 INFO  [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:50,268 DEBUG [M:0;d9f49988d155:44767 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797550129Disabling compacts and flushes for region at 1733797550129Disabling writes for close at 1733797550130 (+1 ms)Obtaining lock to block concurrent updates at 1733797550130Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797550130Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23714, getHeapSize=29768, getOffHeapSize=0, getCellsCount=67 at 1733797550130Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797550141 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797550141Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797550155 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797550155Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797550166 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797550180 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797550180Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797550192 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797550208 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797550208Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797550218 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797550231 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797550231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2eee6b2f: reopening flushed file at 1733797550241 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ac79bf7: reopening flushed file at 1733797550246 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@177b37bc: reopening flushed file at 1733797550253 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bba165c: reopening flushed file at 1733797550259 (+6 ms)Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=56, compaction requested=false at 1733797550266 (+7 ms)Writing region close event to WAL at 1733797550268 (+2 ms)Closed at 1733797550268
2024-12-10T02:25:50,268 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,268 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,269 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,269 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,269 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:25:50,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35443 is added to blk_1073741847_1031 (size=757)
2024-12-10T02:25:50,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41993 is added to blk_1073741847_1031 (size=757)
2024-12-10T02:25:50,272 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:25:51,008 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:51,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:51,226 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,227 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,240 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,241 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,241 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,241 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,241 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,242 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,244 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,244 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,245 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,246 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,250 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,752 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:25:51,753 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,754 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,771 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,772 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,774 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,775 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:51,776 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:25:52,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:52,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:53,009 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:53,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:53,087 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-12-10T02:25:54,010 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:54,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:54,138 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 after 4002ms
2024-12-10T02:25:54,138 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/WALs/d9f49988d155,44767,1733797520290/d9f49988d155%2C44767%2C1733797520290.1733797520430 to hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/oldWALs/d9f49988d155%2C44767%2C1733797520290.1733797520430
2024-12-10T02:25:54,141 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/MasterData/oldWALs/d9f49988d155%2C44767%2C1733797520290.1733797520430 to hdfs://localhost:33395/user/jenkins/test-data/935e7068-1b4b-823e-6694-13292a673eb6/oldWALs/d9f49988d155%2C44767%2C1733797520290.1733797520430$masterlocalwal$
2024-12-10T02:25:54,141 INFO  [M:0;d9f49988d155:44767 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:25:54,141 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:25:54,141 INFO  [M:0;d9f49988d155:44767 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44767
2024-12-10T02:25:54,141 INFO  [M:0;d9f49988d155:44767 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:25:54,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:54,243 INFO  [M:0;d9f49988d155:44767 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:25:54,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44767-0x1019a308e220000, quorum=127.0.0.1:62310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:25:54,245 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@262fed34{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:54,246 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f7ae33f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:54,246 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:54,246 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5091fc79{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:54,246 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39c5f69b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:54,247 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:54,247 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:54,247 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:54,247 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid 7edefdd4-15d7-44d6-870a-075e2b64a7be) service to localhost/127.0.0.1:33395
2024-12-10T02:25:54,248 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data3/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:54,248 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data4/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:54,249 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:54,250 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@362ea065{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:54,251 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78cc0925{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:54,251 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:54,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a382d25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:54,251 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45c55ac5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:54,252 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:25:54,252 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:25:54,252 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:25:54,252 WARN  [BP-1457422663-172.17.0.2-1733797519504 heartbeating to localhost/127.0.0.1:33395 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1457422663-172.17.0.2-1733797519504 (Datanode Uuid eab0cac4-6345-4706-b8fe-8858c6dc8917) service to localhost/127.0.0.1:33395
2024-12-10T02:25:54,252 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data1/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:54,253 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/cluster_70a28b8d-4994-2e04-a593-73e82286675c/data/data2/current/BP-1457422663-172.17.0.2-1733797519504 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:25:54,253 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:25:54,258 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16304a50{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:25:54,259 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e4fac25{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:25:54,259 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:25:54,259 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@347a2271{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:25:54,259 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1915705e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir/,STOPPED}
2024-12-10T02:25:54,265 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:25:54,281 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:25:54,289 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=181 (was 162)
Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:33395
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:33395
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33395 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33395 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33395
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33395 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=455 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=36 (was 27) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3764 (was 3926)
2024-12-10T02:25:54,296 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=181, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=36, ProcessCount=11, AvailableMemoryMB=3764
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.log.dir so I do NOT create it in target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d4108f7d-f821-7edd-5acb-80b773f273f8/hadoop.tmp.dir so I do NOT create it in target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204, deleteOnExit=true
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/test.cache.data in system properties and HBase conf
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:25:54,297 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:25:54,297 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:25:54,298 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:25:54,311 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:25:54,389 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:54,393 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:54,394 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:54,394 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:54,394 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:54,395 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:54,395 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dd613b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:54,396 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42501282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:54,510 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fa01b00{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/java.io.tmpdir/jetty-localhost-36005-hadoop-hdfs-3_4_1-tests_jar-_-any-17983614675606278351/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:25:54,511 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@44b58721{HTTP/1.1, (http/1.1)}{localhost:36005}
2024-12-10T02:25:54,511 INFO  [Time-limited test {}] server.Server(415): Started @183569ms
2024-12-10T02:25:54,523 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:25:54,580 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:54,583 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:54,584 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:54,584 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:54,584 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:54,584 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f08894b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:54,585 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78617008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:54,699 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3fdc15a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/java.io.tmpdir/jetty-localhost-46671-hadoop-hdfs-3_4_1-tests_jar-_-any-6453070247585408211/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:54,699 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8d4c846{HTTP/1.1, (http/1.1)}{localhost:46671}
2024-12-10T02:25:54,699 INFO  [Time-limited test {}] server.Server(415): Started @183758ms
2024-12-10T02:25:54,701 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:54,729 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:25:54,731 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:25:54,732 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:25:54,732 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:25:54,732 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:25:54,733 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:25:54,733 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:25:54,806 WARN  [Thread-1641 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data2/current/BP-320424626-172.17.0.2-1733797554327/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:54,806 WARN  [Thread-1640 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data1/current/BP-320424626-172.17.0.2-1733797554327/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:54,822 WARN  [Thread-1619 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:54,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f09d83107076c20 with lease ID 0x5b769637a541b160: Processing first storage report for DS-359a47d9-5f2b-446a-b096-fc1b1d2d5912 from datanode DatanodeRegistration(127.0.0.1:39019, datanodeUuid=2921afbf-f93e-42b0-bed3-776ea216fbea, infoPort=44383, infoSecurePort=0, ipcPort=42649, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327)
2024-12-10T02:25:54,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f09d83107076c20 with lease ID 0x5b769637a541b160: from storage DS-359a47d9-5f2b-446a-b096-fc1b1d2d5912 node DatanodeRegistration(127.0.0.1:39019, datanodeUuid=2921afbf-f93e-42b0-bed3-776ea216fbea, infoPort=44383, infoSecurePort=0, ipcPort=42649, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T02:25:54,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f09d83107076c20 with lease ID 0x5b769637a541b160: Processing first storage report for DS-ed5dfd9a-c3eb-4034-bc35-fe6c5b971ad5 from datanode DatanodeRegistration(127.0.0.1:39019, datanodeUuid=2921afbf-f93e-42b0-bed3-776ea216fbea, infoPort=44383, infoSecurePort=0, ipcPort=42649, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327)
2024-12-10T02:25:54,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f09d83107076c20 with lease ID 0x5b769637a541b160: from storage DS-ed5dfd9a-c3eb-4034-bc35-fe6c5b971ad5 node DatanodeRegistration(127.0.0.1:39019, datanodeUuid=2921afbf-f93e-42b0-bed3-776ea216fbea, infoPort=44383, infoSecurePort=0, ipcPort=42649, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:54,857 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5381625b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/java.io.tmpdir/jetty-localhost-32797-hadoop-hdfs-3_4_1-tests_jar-_-any-14408987228096530354/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:25:54,858 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:32797}
2024-12-10T02:25:54,858 INFO  [Time-limited test {}] server.Server(415): Started @183916ms
2024-12-10T02:25:54,859 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:25:54,960 WARN  [Thread-1666 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data3/current/BP-320424626-172.17.0.2-1733797554327/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:54,960 WARN  [Thread-1667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data4/current/BP-320424626-172.17.0.2-1733797554327/current, will proceed with Du for space computation calculation, 
2024-12-10T02:25:54,978 WARN  [Thread-1655 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:25:54,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20871a36447d8090 with lease ID 0x5b769637a541b161: Processing first storage report for DS-5a11264c-cca0-481c-b1c2-7742c5443684 from datanode DatanodeRegistration(127.0.0.1:46555, datanodeUuid=29c68f3d-9319-4414-81d4-8f8f8d5c9a91, infoPort=37209, infoSecurePort=0, ipcPort=45767, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327)
2024-12-10T02:25:54,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20871a36447d8090 with lease ID 0x5b769637a541b161: from storage DS-5a11264c-cca0-481c-b1c2-7742c5443684 node DatanodeRegistration(127.0.0.1:46555, datanodeUuid=29c68f3d-9319-4414-81d4-8f8f8d5c9a91, infoPort=37209, infoSecurePort=0, ipcPort=45767, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:54,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x20871a36447d8090 with lease ID 0x5b769637a541b161: Processing first storage report for DS-3e1227ee-4a38-4d86-b8ce-4e8b2aebd72a from datanode DatanodeRegistration(127.0.0.1:46555, datanodeUuid=29c68f3d-9319-4414-81d4-8f8f8d5c9a91, infoPort=37209, infoSecurePort=0, ipcPort=45767, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327)
2024-12-10T02:25:54,981 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x20871a36447d8090 with lease ID 0x5b769637a541b161: from storage DS-3e1227ee-4a38-4d86-b8ce-4e8b2aebd72a node DatanodeRegistration(127.0.0.1:46555, datanodeUuid=29c68f3d-9319-4414-81d4-8f8f8d5c9a91, infoPort=37209, infoSecurePort=0, ipcPort=45767, storageInfo=lv=-57;cid=testClusterID;nsid=689313670;c=1733797554327), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:25:54,987 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36
2024-12-10T02:25:54,990 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/zookeeper_0, clientPort=61029, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:25:54,991 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61029
2024-12-10T02:25:54,991 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:54,992 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:25:55,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:25:55,001 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570 with version=8
2024-12-10T02:25:55,001 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:25:55,003 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:25:55,004 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:25:55,004 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41497
2024-12-10T02:25:55,006 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41497 connecting to ZooKeeper ensemble=127.0.0.1:61029
2024-12-10T02:25:55,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:55,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414970x0, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:25:55,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41497-0x1019a3115bd0000 connected
2024-12-10T02:25:55,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:55,030 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,031 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,033 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:55,033 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570, hbase.cluster.distributed=false
2024-12-10T02:25:55,035 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:25:55,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41497
2024-12-10T02:25:55,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41497
2024-12-10T02:25:55,039 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41497
2024-12-10T02:25:55,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41497
2024-12-10T02:25:55,040 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41497
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:25:55,056 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:25:55,057 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40947
2024-12-10T02:25:55,058 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40947 connecting to ZooKeeper ensemble=127.0.0.1:61029
2024-12-10T02:25:55,059 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,060 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409470x0, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:25:55,066 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:409470x0, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:25:55,066 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40947-0x1019a3115bd0001 connected
2024-12-10T02:25:55,066 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:25:55,067 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:25:55,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:25:55,068 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:25:55,069 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40947
2024-12-10T02:25:55,069 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40947
2024-12-10T02:25:55,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40947
2024-12-10T02:25:55,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40947
2024-12-10T02:25:55,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40947
2024-12-10T02:25:55,084 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:41497
2024-12-10T02:25:55,084 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:55,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:55,086 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:25:55,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,088 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:25:55,089 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,41497,1733797555003 from backup master directory
2024-12-10T02:25:55,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:55,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:25:55,090 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:25:55,090 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,094 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/hbase.id] with ID: 306c3ee1-0f11-4c0b-b81a-914973799cdf
2024-12-10T02:25:55,094 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/.tmp/hbase.id
2024-12-10T02:25:55,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:25:55,102 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:25:55,102 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/.tmp/hbase.id]:[hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/hbase.id]
2024-12-10T02:25:55,112 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,112 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:25:55,113 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T02:25:55,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:25:55,125 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:25:55,126 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:25:55,126 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:25:55,127 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:55,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:25:55,133 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:25:55,134 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store
2024-12-10T02:25:55,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:25:55,139 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:25:55,140 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:55,140 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:25:55,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797555140Disabling compacts and flushes for region at 1733797555140Disabling writes for close at 1733797555140Writing region close event to WAL at 1733797555140Closed at 1733797555140
2024-12-10T02:25:55,141 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/.initializing
2024-12-10T02:25:55,141 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/WALs/d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,143 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C41497%2C1733797555003, suffix=, logDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/WALs/d9f49988d155,41497,1733797555003, archiveDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/oldWALs, maxLogs=10
2024-12-10T02:25:55,143 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C41497%2C1733797555003.1733797555143
2024-12-10T02:25:55,147 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/WALs/d9f49988d155,41497,1733797555003/d9f49988d155%2C41497%2C1733797555003.1733797555143
2024-12-10T02:25:55,148 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44383:44383),(127.0.0.1/127.0.0.1:37209:37209)]
2024-12-10T02:25:55,148 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:55,148 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:55,149 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,149 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,150 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:55,151 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:25:55,151 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,151 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,151 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:55,152 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:25:55,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,153 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:55,153 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:55,154 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:25:55,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,154 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:55,154 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:25:55,155 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:25:55,155 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,156 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:55,156 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,156 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,157 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,158 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,158 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,159 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:25:55,160 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:25:55,162 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:55,162 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754052, jitterRate=-0.04117436707019806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:25:55,163 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797555149Initializing all the Stores at 1733797555149Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555149Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797555150 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797555150Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797555150Cleaning up temporary data from old regions at 1733797555158 (+8 ms)Region opened successfully at 1733797555163 (+5 ms)
2024-12-10T02:25:55,163 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:25:55,166 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573cd32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:25:55,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:25:55,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:25:55,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:25:55,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:25:55,167 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:25:55,168 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:25:55,168 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:25:55,169 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:25:55,170 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:25:55,171 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:25:55,171 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:25:55,172 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:25:55,173 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:25:55,173 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:25:55,174 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:25:55,175 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:25:55,176 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:25:55,178 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:25:55,179 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:25:55,180 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:25:55,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:55,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:25:55,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,182 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,41497,1733797555003, sessionid=0x1019a3115bd0000, setting cluster-up flag (Was=false)
2024-12-10T02:25:55,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,190 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:25:55,191 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,205 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:25:55,206 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,206 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:25:55,208 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:55,208 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:25:55,209 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:25:55,209 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,41497,1733797555003
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:25:55,210 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,212 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797585212
2024-12-10T02:25:55,212 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:25:55,212 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:25:55,212 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:25:55,212 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:25:55,213 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:25:55,213 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:25:55,213 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:25:55,214 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797555214,5,FailOnTimeoutGroup]
2024-12-10T02:25:55,214 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797555214,5,FailOnTimeoutGroup]
2024-12-10T02:25:55,214 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,214 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:25:55,214 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,214 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,214 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,214 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:25:55,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:25:55,223 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:25:55,224 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:25:55,224 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570
2024-12-10T02:25:55,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:25:55,230 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:25:55,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:55,232 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:25:55,233 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:25:55,233 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,234 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,234 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:25:55,235 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:25:55,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,235 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,235 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:25:55,236 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:25:55,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,237 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,237 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:25:55,238 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:25:55,238 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,238 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:25:55,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740
2024-12-10T02:25:55,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740
2024-12-10T02:25:55,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:25:55,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:25:55,241 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:25:55,242 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:25:55,244 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:55,244 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=750153, jitterRate=-0.04613138735294342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:25:55,244 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797555231Initializing all the Stores at 1733797555231Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555232 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555232Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797555232Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555232Cleaning up temporary data from old regions at 1733797555240 (+8 ms)Region opened successfully at 1733797555244 (+4 ms)
2024-12-10T02:25:55,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:25:55,245 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:25:55,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:25:55,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:25:55,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:25:55,245 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:25:55,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797555245Disabling compacts and flushes for region at 1733797555245Disabling writes for close at 1733797555245Writing region close event to WAL at 1733797555245Closed at 1733797555245
2024-12-10T02:25:55,246 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:55,246 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:25:55,246 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:25:55,248 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:25:55,249 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:25:55,274 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(746): ClusterId : 306c3ee1-0f11-4c0b-b81a-914973799cdf
2024-12-10T02:25:55,274 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:25:55,276 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:25:55,276 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:25:55,278 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:25:55,279 DEBUG [RS:0;d9f49988d155:40947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638dd8aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:25:55,290 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:40947
2024-12-10T02:25:55,290 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:25:55,290 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:25:55,290 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:25:55,291 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,41497,1733797555003 with port=40947, startcode=1733797555055
2024-12-10T02:25:55,291 DEBUG [RS:0;d9f49988d155:40947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:25:55,293 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60267, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:25:55,294 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41497 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,294 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41497 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,295 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570
2024-12-10T02:25:55,296 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44473
2024-12-10T02:25:55,296 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:25:55,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:25:55,297 DEBUG [RS:0;d9f49988d155:40947 {}] zookeeper.ZKUtil(111): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,298 WARN  [RS:0;d9f49988d155:40947 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:25:55,298 INFO  [RS:0;d9f49988d155:40947 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:55,298 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,298 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,40947,1733797555055]
2024-12-10T02:25:55,301 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:25:55,302 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:25:55,302 INFO  [RS:0;d9f49988d155:40947 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:25:55,302 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,303 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:25:55,303 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:25:55,303 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:25:55,304 DEBUG [RS:0;d9f49988d155:40947 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,307 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,40947,1733797555055-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:25:55,321 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:25:55,321 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,40947,1733797555055-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,321 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,321 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.Replication(171): d9f49988d155,40947,1733797555055 started
2024-12-10T02:25:55,335 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,335 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,40947,1733797555055, RpcServer on d9f49988d155/172.17.0.2:40947, sessionid=0x1019a3115bd0001
2024-12-10T02:25:55,335 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:25:55,335 DEBUG [RS:0;d9f49988d155:40947 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,335 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,40947,1733797555055'
2024-12-10T02:25:55,335 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,40947,1733797555055'
2024-12-10T02:25:55,336 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:25:55,337 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:25:55,337 DEBUG [RS:0;d9f49988d155:40947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:25:55,337 INFO  [RS:0;d9f49988d155:40947 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:25:55,337 INFO  [RS:0;d9f49988d155:40947 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:25:55,399 WARN  [d9f49988d155:41497 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:25:55,439 INFO  [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C40947%2C1733797555055, suffix=, logDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055, archiveDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs, maxLogs=32
2024-12-10T02:25:55,439 INFO  [RS:0;d9f49988d155:40947 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40947%2C1733797555055.1733797555439
2024-12-10T02:25:55,444 INFO  [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797555439
2024-12-10T02:25:55,445 DEBUG [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37209:37209),(127.0.0.1/127.0.0.1:44383:44383)]
2024-12-10T02:25:55,649 DEBUG [d9f49988d155:41497 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:25:55,650 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,651 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,40947,1733797555055, state=OPENING
2024-12-10T02:25:55,654 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:25:55,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:25:55,656 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:25:55,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:55,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:55,657 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,40947,1733797555055}]
2024-12-10T02:25:55,810 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:25:55,811 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47719, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:25:55,815 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:25:55,815 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:25:55,817 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C40947%2C1733797555055.meta, suffix=.meta, logDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055, archiveDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs, maxLogs=32
2024-12-10T02:25:55,818 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40947%2C1733797555055.meta.1733797555818.meta
2024-12-10T02:25:55,825 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.meta.1733797555818.meta
2024-12-10T02:25:55,826 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44383:44383),(127.0.0.1/127.0.0.1:37209:37209)]
2024-12-10T02:25:55,827 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:55,827 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:25:55,827 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:25:55,828 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:25:55,828 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:25:55,828 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:55,828 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:25:55,828 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:25:55,829 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:25:55,830 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:25:55,830 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,831 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:25:55,831 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:25:55,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,832 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,832 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:25:55,832 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:25:55,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,833 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,833 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:25:55,833 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:25:55,833 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,834 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:25:55,834 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:25:55,835 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740
2024-12-10T02:25:55,836 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740
2024-12-10T02:25:55,837 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:25:55,837 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:25:55,837 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:25:55,838 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:25:55,839 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784988, jitterRate=-0.0018363147974014282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:25:55,839 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:25:55,840 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797555828Writing region info on filesystem at 1733797555828Initializing all the Stores at 1733797555829 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555829Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797555829Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797555829Cleaning up temporary data from old regions at 1733797555837 (+8 ms)Running coprocessor post-open hooks at 1733797555839 (+2 ms)Region opened successfully at 1733797555840 (+1 ms)
2024-12-10T02:25:55,841 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797555809
2024-12-10T02:25:55,843 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:25:55,843 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:25:55,844 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,845 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,40947,1733797555055, state=OPEN
2024-12-10T02:25:55,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:25:55,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:25:55,850 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,40947,1733797555055
2024-12-10T02:25:55,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:55,850 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:25:55,853 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:25:55,853 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,40947,1733797555055 in 194 msec
2024-12-10T02:25:55,855 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:25:55,856 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec
2024-12-10T02:25:55,856 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:25:55,856 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:25:55,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:25:55,858 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,40947,1733797555055, seqNum=-1]
2024-12-10T02:25:55,858 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:25:55,860 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44993, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:25:55,865 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 656 msec
2024-12-10T02:25:55,865 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797555865, completionTime=-1
2024-12-10T02:25:55,865 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:25:55,866 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797615868
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797675868
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,868 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,869 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:41497, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,869 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,869 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,870 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:25:55,872 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.782sec
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:25:55,873 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:25:55,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c1e7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:55,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,41497,-1 for getting cluster id
2024-12-10T02:25:55,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:25:55,875 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:25:55,875 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:25:55,875 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,41497,1733797555003-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:25:55,877 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '306c3ee1-0f11-4c0b-b81a-914973799cdf'
2024-12-10T02:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "306c3ee1-0f11-4c0b-b81a-914973799cdf"

2024-12-10T02:25:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd81cf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,41497,-1]
2024-12-10T02:25:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:25:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:25:55,879 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:25:55,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf4f1e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:25:55,880 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:25:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,40947,1733797555055, seqNum=-1]
2024-12-10T02:25:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:25:55,882 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:25:55,884 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,884 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:25:55,886 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:25:55,886 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T02:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is d9f49988d155,41497,1733797555003
2024-12-10T02:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c5288a6
2024-12-10T02:25:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T02:25:55,888 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T02:25:55,889 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T02:25:55,889 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T02:25:55,889 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:25:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:25:55,891 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T02:25:55,891 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:55,891 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-12-10T02:25:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:25:55,893 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T02:25:55,898 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741835_1011 (size=405)
2024-12-10T02:25:55,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741835_1011 (size=405)
2024-12-10T02:25:55,900 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64c03f10bab53cdf0aabd97b04da4f2f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570
2024-12-10T02:25:55,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741836_1012 (size=88)
2024-12-10T02:25:55,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741836_1012 (size=88)
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 64c03f10bab53cdf0aabd97b04da4f2f, disabling compactions & flushes
2024-12-10T02:25:55,907 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f. after waiting 0 ms
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:55,907 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:55,907 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64c03f10bab53cdf0aabd97b04da4f2f:
Waiting for close lock at 1733797555907Disabling compacts and flushes for region at 1733797555907Disabling writes for close at 1733797555907Writing region close event to WAL at 1733797555907Closed at 1733797555907
2024-12-10T02:25:55,909 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T02:25:55,909 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1733797555909"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797555909"}]},"ts":"1733797555909"}
2024-12-10T02:25:55,911 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T02:25:55,912 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T02:25:55,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797555912"}]},"ts":"1733797555912"}
2024-12-10T02:25:55,915 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-12-10T02:25:55,915 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c03f10bab53cdf0aabd97b04da4f2f, ASSIGN}]
2024-12-10T02:25:55,916 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c03f10bab53cdf0aabd97b04da4f2f, ASSIGN
2024-12-10T02:25:55,917 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c03f10bab53cdf0aabd97b04da4f2f, ASSIGN; state=OFFLINE, location=d9f49988d155,40947,1733797555055; forceNewPlan=false, retain=false
2024-12-10T02:25:56,011 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:56,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:56,068 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64c03f10bab53cdf0aabd97b04da4f2f, regionState=OPENING, regionLocation=d9f49988d155,40947,1733797555055
2024-12-10T02:25:56,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c03f10bab53cdf0aabd97b04da4f2f, ASSIGN because future has completed
2024-12-10T02:25:56,071 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c03f10bab53cdf0aabd97b04da4f2f, server=d9f49988d155,40947,1733797555055}]
2024-12-10T02:25:56,228 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:56,228 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 64c03f10bab53cdf0aabd97b04da4f2f, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:25:56,228 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,228 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:25:56,228 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,228 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,230 INFO  [StoreOpener-64c03f10bab53cdf0aabd97b04da4f2f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 64c03f10bab53cdf0aabd97b04da4f2f 
2024-12-10T02:25:56,231 INFO  [StoreOpener-64c03f10bab53cdf0aabd97b04da4f2f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64c03f10bab53cdf0aabd97b04da4f2f columnFamilyName info
2024-12-10T02:25:56,231 DEBUG [StoreOpener-64c03f10bab53cdf0aabd97b04da4f2f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:25:56,231 INFO  [StoreOpener-64c03f10bab53cdf0aabd97b04da4f2f-1 {}] regionserver.HStore(327): Store=64c03f10bab53cdf0aabd97b04da4f2f/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:25:56,231 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,232 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,232 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,233 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,233 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,234 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:25:56,234 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-12-10T02:25:56,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:25:56,235 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-12-10T02:25:56,236 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:25:56,236 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 64c03f10bab53cdf0aabd97b04da4f2f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=777267, jitterRate=-0.011655166745185852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:25:56,236 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:25:56,237 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 64c03f10bab53cdf0aabd97b04da4f2f:
Running coprocessor pre-open hook at 1733797556229Writing region info on filesystem at 1733797556229Initializing all the Stores at 1733797556229Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797556229Cleaning up temporary data from old regions at 1733797556233 (+4 ms)Running coprocessor post-open hooks at 1733797556236 (+3 ms)Region opened successfully at 1733797556237 (+1 ms)
2024-12-10T02:25:56,238 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f., pid=6, masterSystemTime=1733797556224
2024-12-10T02:25:56,240 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:56,240 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:25:56,241 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64c03f10bab53cdf0aabd97b04da4f2f, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,40947,1733797555055
2024-12-10T02:25:56,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c03f10bab53cdf0aabd97b04da4f2f, server=d9f49988d155,40947,1733797555055 because future has completed
2024-12-10T02:25:56,247 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T02:25:56,247 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 64c03f10bab53cdf0aabd97b04da4f2f, server=d9f49988d155,40947,1733797555055 in 174 msec
2024-12-10T02:25:56,249 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T02:25:56,250 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c03f10bab53cdf0aabd97b04da4f2f, ASSIGN in 332 msec
2024-12-10T02:25:56,250 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T02:25:56,251 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797556250"}]},"ts":"1733797556250"}
2024-12-10T02:25:56,253 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-12-10T02:25:56,253 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T02:25:56,255 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 364 msec
2024-12-10T02:25:57,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:57,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:58,012 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:58,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:59,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:25:59,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:00,013 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:00,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:01,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:01,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:01,329 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:26:01,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,331 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,348 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,349 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,352 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,353 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,355 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:01,359 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:26:01,360 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-12-10T02:26:02,014 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:02,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:03,015 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:03,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:04,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:04,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:05,016 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:05,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:26:05,975 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T02:26:05,975 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-12-10T02:26:05,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:05,978 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f., hostname=d9f49988d155,40947,1733797555055, seqNum=2]
2024-12-10T02:26:05,988 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:05,994 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T02:26:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-10T02:26:05,995 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T02:26:05,996 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T02:26:06,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:06,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:06,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-12-10T02:26:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:06,157 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 64c03f10bab53cdf0aabd97b04da4f2f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:26:06,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/f92c859668904ed2ae891988055a2957 is 1080, key is row0001/info:/1733797565982/Put/seqid=0
2024-12-10T02:26:06,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741837_1013 (size=6033)
2024-12-10T02:26:06,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741837_1013 (size=6033)
2024-12-10T02:26:06,180 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/f92c859668904ed2ae891988055a2957
2024-12-10T02:26:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/f92c859668904ed2ae891988055a2957 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957
2024-12-10T02:26:06,191 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957, entries=1, sequenceid=5, filesize=5.9 K
2024-12-10T02:26:06,192 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 34ms, sequenceid=5, compaction requested=false
2024-12-10T02:26:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 64c03f10bab53cdf0aabd97b04da4f2f:

2024-12-10T02:26:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:06,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-12-10T02:26:06,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-12-10T02:26:06,199 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-10T02:26:06,199 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 200 msec
2024-12-10T02:26:06,201 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 211 msec
2024-12-10T02:26:06,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:06,235 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T02:26:07,017 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:07,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:08,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:08,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:09,018 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:09,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:10,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:10,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:11,019 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:11,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:12,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:12,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:13,020 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:13,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:14,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:14,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:15,021 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:15,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:16,022 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:16,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-12-10T02:26:16,074 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T02:26:16,077 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-10T02:26:16,080 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T02:26:16,081 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T02:26:16,081 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T02:26:16,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-12-10T02:26:16,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:16,235 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 64c03f10bab53cdf0aabd97b04da4f2f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:26:16,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/498957c1c0d34c23b33b94d42e0f69d1 is 1080, key is row0002/info:/1733797576076/Put/seqid=0
2024-12-10T02:26:16,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741838_1014 (size=6033)
2024-12-10T02:26:16,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741838_1014 (size=6033)
2024-12-10T02:26:16,245 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/498957c1c0d34c23b33b94d42e0f69d1
2024-12-10T02:26:16,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/498957c1c0d34c23b33b94d42e0f69d1 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1
2024-12-10T02:26:16,255 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1, entries=1, sequenceid=9, filesize=5.9 K
2024-12-10T02:26:16,256 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 21ms, sequenceid=9, compaction requested=false
2024-12-10T02:26:16,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 64c03f10bab53cdf0aabd97b04da4f2f:

2024-12-10T02:26:16,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:16,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-12-10T02:26:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-12-10T02:26:16,260 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-12-10T02:26:16,260 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-12-10T02:26:16,263 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-12-10T02:26:17,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:17,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:18,023 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:18,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:19,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:19,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:20,024 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:20,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:21,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:21,025 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224 after 68044ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:26:21,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:21,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta after 68032ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-10T02:26:22,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:22,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:23,026 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:23,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:24,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:24,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:24,987 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:26:25,027 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:25,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:26,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:26,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-12-10T02:26:26,135 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T02:26:26,139 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40947%2C1733797555055.1733797586139
2024-12-10T02:26:26,145 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:26,145 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:26,145 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:26,145 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:26,145 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:26,146 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797555439 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797586139
2024-12-10T02:26:26,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37209:37209),(127.0.0.1/127.0.0.1:44383:44383)]
2024-12-10T02:26:26,148 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797555439 is not closed yet, will try archiving it next time
2024-12-10T02:26:26,149 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:26,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741833_1009 (size=5546)
2024-12-10T02:26:26,149 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741833_1009 (size=5546)
2024-12-10T02:26:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-10T02:26:26,151 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T02:26:26,152 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T02:26:26,152 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T02:26:26,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-12-10T02:26:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:26,306 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 64c03f10bab53cdf0aabd97b04da4f2f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:26:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/1ef1156824b245a48cd26ca082658adf is 1080, key is row0003/info:/1733797586136/Put/seqid=0
2024-12-10T02:26:26,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741840_1016 (size=6033)
2024-12-10T02:26:26,315 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741840_1016 (size=6033)
2024-12-10T02:26:26,315 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/1ef1156824b245a48cd26ca082658adf
2024-12-10T02:26:26,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/1ef1156824b245a48cd26ca082658adf as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf
2024-12-10T02:26:26,327 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf, entries=1, sequenceid=13, filesize=5.9 K
2024-12-10T02:26:26,328 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 22ms, sequenceid=13, compaction requested=true
2024-12-10T02:26:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 64c03f10bab53cdf0aabd97b04da4f2f:

2024-12-10T02:26:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:26,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-12-10T02:26:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-12-10T02:26:26,332 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-12-10T02:26:26,333 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec
2024-12-10T02:26:26,335 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-12-10T02:26:27,028 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:27,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:28,029 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:28,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:29,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:29,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:30,030 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:30,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:31,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:31,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:32,031 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:32,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:33,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:33,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:34,032 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:34,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:35,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:35,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:36,033 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:36,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-12-10T02:26:36,215 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T02:26:36,215 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:26:36,216 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:26:36,216 DEBUG [Time-limited test {}] regionserver.HStore(1541): 64c03f10bab53cdf0aabd97b04da4f2f/info is initiating minor compaction (all files)
2024-12-10T02:26:36,216 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:26:36,216 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:36,217 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 64c03f10bab53cdf0aabd97b04da4f2f/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:36,217 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf] into tmpdir=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp, totalSize=17.7 K
2024-12-10T02:26:36,217 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f92c859668904ed2ae891988055a2957, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733797565982
2024-12-10T02:26:36,217 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 498957c1c0d34c23b33b94d42e0f69d1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1733797576076
2024-12-10T02:26:36,218 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1ef1156824b245a48cd26ca082658adf, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733797586136
2024-12-10T02:26:36,228 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 64c03f10bab53cdf0aabd97b04da4f2f#info#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:26:36,229 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/f7aa2a7240fa43e4998e65b92836d6b4 is 1080, key is row0001/info:/1733797565982/Put/seqid=0
2024-12-10T02:26:36,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741841_1017 (size=8296)
2024-12-10T02:26:36,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741841_1017 (size=8296)
2024-12-10T02:26:36,593 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T02:26:36,593 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T02:26:36,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/f7aa2a7240fa43e4998e65b92836d6b4 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f7aa2a7240fa43e4998e65b92836d6b4
2024-12-10T02:26:36,649 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64c03f10bab53cdf0aabd97b04da4f2f/info of 64c03f10bab53cdf0aabd97b04da4f2f into f7aa2a7240fa43e4998e65b92836d6b4(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:26:36,649 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 64c03f10bab53cdf0aabd97b04da4f2f:

2024-12-10T02:26:36,651 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40947%2C1733797555055.1733797596651
2024-12-10T02:26:36,657 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:36,657 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:36,657 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:36,657 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:36,657 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:36,658 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797586139 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797596651
2024-12-10T02:26:36,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741839_1015 (size=2520)
2024-12-10T02:26:36,659 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741839_1015 (size=2520)
2024-12-10T02:26:36,664 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44383:44383),(127.0.0.1/127.0.0.1:37209:37209)]
2024-12-10T02:26:36,664 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797555439 to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs/d9f49988d155%2C40947%2C1733797555055.1733797555439
2024-12-10T02:26:36,665 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-10T02:26:36,667 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-12-10T02:26:36,668 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-12-10T02:26:36,668 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-12-10T02:26:36,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-12-10T02:26:36,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:36,821 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 64c03f10bab53cdf0aabd97b04da4f2f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:26:36,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/c73e22bc009f4340957f8499e7045bdc is 1080, key is row0000/info:/1733797596650/Put/seqid=0
2024-12-10T02:26:36,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741843_1019 (size=6033)
2024-12-10T02:26:36,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741843_1019 (size=6033)
2024-12-10T02:26:36,831 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/c73e22bc009f4340957f8499e7045bdc
2024-12-10T02:26:36,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/c73e22bc009f4340957f8499e7045bdc as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/c73e22bc009f4340957f8499e7045bdc
2024-12-10T02:26:36,842 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/c73e22bc009f4340957f8499e7045bdc, entries=1, sequenceid=18, filesize=5.9 K
2024-12-10T02:26:36,844 INFO  [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 22ms, sequenceid=18, compaction requested=false
2024-12-10T02:26:36,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 64c03f10bab53cdf0aabd97b04da4f2f:

2024-12-10T02:26:36,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:36,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-12-10T02:26:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-12-10T02:26:36,848 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-12-10T02:26:36,848 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-12-10T02:26:36,850 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-12-10T02:26:37,034 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:37,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:38,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:38,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:39,035 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:39,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:40,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:40,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:41,036 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:41,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:41,228 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 64c03f10bab53cdf0aabd97b04da4f2f, had cached 0 bytes from a total of 14329
2024-12-10T02:26:42,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:42,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:43,037 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:43,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:44,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:44,041 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:45,038 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:45,041 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:46,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:46,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41497 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-12-10T02:26:46,735 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-12-10T02:26:46,738 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C40947%2C1733797555055.1733797606737
2024-12-10T02:26:46,744 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,744 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,744 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,744 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,744 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,744 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797596651 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797606737
2024-12-10T02:26:46,745 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44383:44383),(127.0.0.1/127.0.0.1:37209:37209)]
2024-12-10T02:26:46,745 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797596651 is not closed yet, will try archiving it next time
2024-12-10T02:26:46,745 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:26:46,745 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/WALs/d9f49988d155,40947,1733797555055/d9f49988d155%2C40947%2C1733797555055.1733797586139 to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs/d9f49988d155%2C40947%2C1733797555055.1733797586139
2024-12-10T02:26:46,745 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:26:46,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:26:46,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:46,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:46,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:26:46,746 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:26:46,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1686605989, stopped=false
2024-12-10T02:26:46,746 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,41497,1733797555003
2024-12-10T02:26:46,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741842_1018 (size=2026)
2024-12-10T02:26:46,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741842_1018 (size=2026)
2024-12-10T02:26:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:26:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:26:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:46,748 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:26:46,748 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:26:46,748 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:26:46,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:46,748 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,40947,1733797555055' *****
2024-12-10T02:26:46,748 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:26:46,748 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:26:46,748 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:26:46,749 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(3091): Received CLOSE for 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,40947,1733797555055
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:40947.
2024-12-10T02:26:46,749 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64c03f10bab53cdf0aabd97b04da4f2f, disabling compactions & flushes
2024-12-10T02:26:46,749 DEBUG [RS:0;d9f49988d155:40947 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:26:46,749 DEBUG [RS:0;d9f49988d155:40947 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:46,749 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:46,749 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:46,749 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f. after waiting 0 ms
2024-12-10T02:26:46,749 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:26:46,749 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:26:46,750 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:26:46,750 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:26:46,750 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 64c03f10bab53cdf0aabd97b04da4f2f 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:26:46,750 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-10T02:26:46,750 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 64c03f10bab53cdf0aabd97b04da4f2f=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.}
2024-12-10T02:26:46,750 DEBUG [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 64c03f10bab53cdf0aabd97b04da4f2f
2024-12-10T02:26:46,750 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:26:46,750 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:26:46,750 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:26:46,750 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:26:46,750 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:26:46,750 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-12-10T02:26:46,755 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/0b0b6063fff54d4a8c7ee9103ac29e54 is 1080, key is row0001/info:/1733797606736/Put/seqid=0
2024-12-10T02:26:46,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741845_1021 (size=6033)
2024-12-10T02:26:46,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741845_1021 (size=6033)
2024-12-10T02:26:46,760 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/0b0b6063fff54d4a8c7ee9103ac29e54
2024-12-10T02:26:46,767 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/.tmp/info/0b0b6063fff54d4a8c7ee9103ac29e54 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/0b0b6063fff54d4a8c7ee9103ac29e54
2024-12-10T02:26:46,768 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/info/febe70b213844bb78e95b5009f0d100a is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f./info:regioninfo/1733797556241/Put/seqid=0
2024-12-10T02:26:46,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741846_1022 (size=7308)
2024-12-10T02:26:46,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741846_1022 (size=7308)
2024-12-10T02:26:46,773 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/0b0b6063fff54d4a8c7ee9103ac29e54, entries=1, sequenceid=22, filesize=5.9 K
2024-12-10T02:26:46,773 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/info/febe70b213844bb78e95b5009f0d100a
2024-12-10T02:26:46,774 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 25ms, sequenceid=22, compaction requested=true
2024-12-10T02:26:46,774 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf] to archive
2024-12-10T02:26:46,775 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:26:46,777 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957 to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/f92c859668904ed2ae891988055a2957
2024-12-10T02:26:46,778 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1 to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/498957c1c0d34c23b33b94d42e0f69d1
2024-12-10T02:26:46,780 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf to hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/info/1ef1156824b245a48cd26ca082658adf
2024-12-10T02:26:46,780 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d9f49988d155:41497 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T02:26:46,781 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [f92c859668904ed2ae891988055a2957=6033, 498957c1c0d34c23b33b94d42e0f69d1=6033, 1ef1156824b245a48cd26ca082658adf=6033]
2024-12-10T02:26:46,785 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c03f10bab53cdf0aabd97b04da4f2f/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-12-10T02:26:46,785 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:46,786 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64c03f10bab53cdf0aabd97b04da4f2f:
Waiting for close lock at 1733797606749Running coprocessor pre-close hooks at 1733797606749Disabling compacts and flushes for region at 1733797606749Disabling writes for close at 1733797606749Obtaining lock to block concurrent updates at 1733797606750 (+1 ms)Preparing flush snapshotting stores in 64c03f10bab53cdf0aabd97b04da4f2f at 1733797606750Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1733797606750Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f. at 1733797606750Flushing 64c03f10bab53cdf0aabd97b04da4f2f/info: creating writer at 1733797606751 (+1 ms)Flushing 64c03f10bab53cdf0aabd97b04da4f2f/info: appending metadata at 1733797606755 (+4 ms)Flushing 64c03f10bab53cdf0aabd97b04da4f2f/info: closing flushed file at 1733797606755Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ec45515: reopening flushed file at 1733797606766 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c03f10bab53cdf0aabd97b04da4f2f in 25ms, sequenceid=22, compaction requested=true at 1733797606774 (+8 ms)Writing region close event to WAL at 1733797606781 (+7 ms)Running coprocessor post-close hooks at 1733797606785 (+4 ms)Closed at 1733797606785
2024-12-10T02:26:46,786 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1733797555889.64c03f10bab53cdf0aabd97b04da4f2f.
2024-12-10T02:26:46,793 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/ns/6a8baf6dd373435db9b9cd4e2e251daa is 43, key is default/ns:d/1733797555860/Put/seqid=0
2024-12-10T02:26:46,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741847_1023 (size=5153)
2024-12-10T02:26:46,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741847_1023 (size=5153)
2024-12-10T02:26:46,797 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/ns/6a8baf6dd373435db9b9cd4e2e251daa
2024-12-10T02:26:46,816 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/table/0202ccb6f24f483681cdc2679e398f25 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1733797556250/Put/seqid=0
2024-12-10T02:26:46,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741848_1024 (size=5508)
2024-12-10T02:26:46,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741848_1024 (size=5508)
2024-12-10T02:26:46,822 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/table/0202ccb6f24f483681cdc2679e398f25
2024-12-10T02:26:46,827 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/info/febe70b213844bb78e95b5009f0d100a as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/info/febe70b213844bb78e95b5009f0d100a
2024-12-10T02:26:46,833 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/info/febe70b213844bb78e95b5009f0d100a, entries=10, sequenceid=11, filesize=7.1 K
2024-12-10T02:26:46,834 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/ns/6a8baf6dd373435db9b9cd4e2e251daa as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/ns/6a8baf6dd373435db9b9cd4e2e251daa
2024-12-10T02:26:46,840 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/ns/6a8baf6dd373435db9b9cd4e2e251daa, entries=2, sequenceid=11, filesize=5.0 K
2024-12-10T02:26:46,841 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/.tmp/table/0202ccb6f24f483681cdc2679e398f25 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/table/0202ccb6f24f483681cdc2679e398f25
2024-12-10T02:26:46,846 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/table/0202ccb6f24f483681cdc2679e398f25, entries=2, sequenceid=11, filesize=5.4 K
2024-12-10T02:26:46,847 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=11, compaction requested=false
2024-12-10T02:26:46,852 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-10T02:26:46,852 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:26:46,852 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:26:46,853 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797606750Running coprocessor pre-close hooks at 1733797606750Disabling compacts and flushes for region at 1733797606750Disabling writes for close at 1733797606750Obtaining lock to block concurrent updates at 1733797606750Preparing flush snapshotting stores in 1588230740 at 1733797606750Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1733797606750Flushing stores of hbase:meta,,1.1588230740 at 1733797606751 (+1 ms)Flushing 1588230740/info: creating writer at 1733797606751Flushing 1588230740/info: appending metadata at 1733797606767 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733797606767Flushing 1588230740/ns: creating writer at 1733797606778 (+11 ms)Flushing 1588230740/ns: appending metadata at 1733797606793 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733797606793Flushing 1588230740/table: creating writer at 1733797606802 (+9 ms)Flushing 1588230740/table: appending metadata at 1733797606816 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733797606816Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5681be78: reopening flushed file at 1733797606827 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41349d57: reopening flushed file at 1733797606833 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13727ae3: reopening flushed file at 1733797606840 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=11, compaction requested=false at 1733797606847 (+7 ms)Writing region close event to WAL at 1733797606848 (+1 ms)Running coprocessor post-close hooks at 1733797606852 (+4 ms)Closed at 1733797606852
2024-12-10T02:26:46,853 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:26:46,950 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,40947,1733797555055; all regions closed.
2024-12-10T02:26:46,951 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,951 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,951 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,951 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,951 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,953 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741834_1010 (size=3306)
2024-12-10T02:26:46,953 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741834_1010 (size=3306)
2024-12-10T02:26:46,956 DEBUG [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs
2024-12-10T02:26:46,956 INFO  [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C40947%2C1733797555055.meta:.meta(num 1733797555818)
2024-12-10T02:26:46,956 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,956 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,956 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,956 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,957 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:46,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741844_1020 (size=1252)
2024-12-10T02:26:46,958 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741844_1020 (size=1252)
2024-12-10T02:26:46,962 DEBUG [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/oldWALs
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C40947%2C1733797555055:(num 1733797606737)
2024-12-10T02:26:46,962 DEBUG [RS:0;d9f49988d155:40947 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:26:46,962 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:26:46,962 INFO  [RS:0;d9f49988d155:40947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40947
2024-12-10T02:26:46,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,40947,1733797555055
2024-12-10T02:26:46,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:26:46,965 INFO  [RS:0;d9f49988d155:40947 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:26:46,965 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,40947,1733797555055]
2024-12-10T02:26:46,969 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,40947,1733797555055 already deleted, retry=false
2024-12-10T02:26:46,969 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,40947,1733797555055 expired; onlineServers=0
2024-12-10T02:26:46,969 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,41497,1733797555003' *****
2024-12-10T02:26:46,969 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:26:46,969 DEBUG [M:0;d9f49988d155:41497 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:26:46,969 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:26:46,969 DEBUG [M:0;d9f49988d155:41497 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:26:46,969 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797555214 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797555214,5,FailOnTimeoutGroup]
2024-12-10T02:26:46,969 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797555214 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797555214,5,FailOnTimeoutGroup]
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:26:46,969 DEBUG [M:0;d9f49988d155:41497 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:26:46,969 INFO  [M:0;d9f49988d155:41497 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:26:46,970 INFO  [M:0;d9f49988d155:41497 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:26:46,970 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:26:46,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:26:46,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:46,971 DEBUG [M:0;d9f49988d155:41497 {}] zookeeper.ZKUtil(347): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:26:46,971 WARN  [M:0;d9f49988d155:41497 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:26:46,971 INFO  [M:0;d9f49988d155:41497 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/.lastflushedseqids
2024-12-10T02:26:46,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741849_1025 (size=130)
2024-12-10T02:26:46,976 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741849_1025 (size=130)
2024-12-10T02:26:46,976 INFO  [M:0;d9f49988d155:41497 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:26:46,977 INFO  [M:0;d9f49988d155:41497 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:26:46,977 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:26:46,977 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:46,977 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:46,977 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:26:46,977 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:46,977 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.54 KB heapSize=54.89 KB
2024-12-10T02:26:46,993 DEBUG [M:0;d9f49988d155:41497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8823441c4d1b42c98d9fb1f9b4ae4943 is 82, key is hbase:meta,,1/info:regioninfo/1733797555844/Put/seqid=0
2024-12-10T02:26:46,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741850_1026 (size=5672)
2024-12-10T02:26:46,997 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741850_1026 (size=5672)
2024-12-10T02:26:46,998 INFO  [M:0;d9f49988d155:41497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8823441c4d1b42c98d9fb1f9b4ae4943
2024-12-10T02:26:47,017 DEBUG [M:0;d9f49988d155:41497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c703c2f6299642c6997df485e411e51b is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733797556254/Put/seqid=0
2024-12-10T02:26:47,021 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741851_1027 (size=7817)
2024-12-10T02:26:47,022 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741851_1027 (size=7817)
2024-12-10T02:26:47,022 INFO  [M:0;d9f49988d155:41497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.94 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c703c2f6299642c6997df485e411e51b
2024-12-10T02:26:47,026 INFO  [M:0;d9f49988d155:41497 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c703c2f6299642c6997df485e411e51b
2024-12-10T02:26:47,039 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:47,041 DEBUG [M:0;d9f49988d155:41497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97897c561c8a4df8a6d4466a1dc6a341 is 69, key is d9f49988d155,40947,1733797555055/rs:state/1733797555294/Put/seqid=0
2024-12-10T02:26:47,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:47,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741852_1028 (size=5156)
2024-12-10T02:26:47,046 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741852_1028 (size=5156)
2024-12-10T02:26:47,046 INFO  [M:0;d9f49988d155:41497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97897c561c8a4df8a6d4466a1dc6a341
2024-12-10T02:26:47,065 DEBUG [M:0;d9f49988d155:41497 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf3b9826375947dd837726e91cf3083a is 52, key is load_balancer_on/state:d/1733797555885/Put/seqid=0
2024-12-10T02:26:47,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:26:47,068 INFO  [RS:0;d9f49988d155:40947 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:26:47,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40947-0x1019a3115bd0001, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:26:47,068 INFO  [RS:0;d9f49988d155:40947 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,40947,1733797555055; zookeeper connection closed.
2024-12-10T02:26:47,068 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@491ef1ad {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@491ef1ad
2024-12-10T02:26:47,069 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:26:47,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741853_1029 (size=5056)
2024-12-10T02:26:47,069 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741853_1029 (size=5056)
2024-12-10T02:26:47,069 INFO  [M:0;d9f49988d155:41497 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf3b9826375947dd837726e91cf3083a
2024-12-10T02:26:47,074 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8823441c4d1b42c98d9fb1f9b4ae4943 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8823441c4d1b42c98d9fb1f9b4ae4943
2024-12-10T02:26:47,078 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8823441c4d1b42c98d9fb1f9b4ae4943, entries=8, sequenceid=121, filesize=5.5 K
2024-12-10T02:26:47,079 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c703c2f6299642c6997df485e411e51b as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c703c2f6299642c6997df485e411e51b
2024-12-10T02:26:47,082 INFO  [M:0;d9f49988d155:41497 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c703c2f6299642c6997df485e411e51b
2024-12-10T02:26:47,083 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c703c2f6299642c6997df485e411e51b, entries=14, sequenceid=121, filesize=7.6 K
2024-12-10T02:26:47,083 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97897c561c8a4df8a6d4466a1dc6a341 as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97897c561c8a4df8a6d4466a1dc6a341
2024-12-10T02:26:47,087 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97897c561c8a4df8a6d4466a1dc6a341, entries=1, sequenceid=121, filesize=5.0 K
2024-12-10T02:26:47,088 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bf3b9826375947dd837726e91cf3083a as hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf3b9826375947dd837726e91cf3083a
2024-12-10T02:26:47,092 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44473/user/jenkins/test-data/998e4e97-b56c-05f0-b529-e3fe4236b570/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bf3b9826375947dd837726e91cf3083a, entries=1, sequenceid=121, filesize=4.9 K
2024-12-10T02:26:47,093 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=121, compaction requested=false
2024-12-10T02:26:47,094 INFO  [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:47,094 DEBUG [M:0;d9f49988d155:41497 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797606977Disabling compacts and flushes for region at 1733797606977Disabling writes for close at 1733797606977Obtaining lock to block concurrent updates at 1733797606977Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797606977Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44584, getHeapSize=56144, getOffHeapSize=0, getCellsCount=140 at 1733797606977Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797606978 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797606978Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797606992 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797606992Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797607002 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797607016 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797607016Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797607027 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797607040 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797607040Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797607050 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797607064 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797607064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@526fa559: reopening flushed file at 1733797607073 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@311dd20: reopening flushed file at 1733797607078 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b92d9d3: reopening flushed file at 1733797607083 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24b0bff0: reopening flushed file at 1733797607087 (+4 ms)Finished flush of dataSize ~43.54 KB/44584, heapSize ~54.83 KB/56144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 115ms, sequenceid=121, compaction requested=false at 1733797607093 (+6 ms)Writing region close event to WAL at 1733797607094 (+1 ms)Closed at 1733797607094
2024-12-10T02:26:47,094 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:47,094 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:47,094 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:47,095 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:47,095 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:26:47,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46555 is added to blk_1073741830_1006 (size=52981)
2024-12-10T02:26:47,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39019 is added to blk_1073741830_1006 (size=52981)
2024-12-10T02:26:47,097 INFO  [M:0;d9f49988d155:41497 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:26:47,097 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:26:47,097 INFO  [M:0;d9f49988d155:41497 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41497
2024-12-10T02:26:47,097 INFO  [M:0;d9f49988d155:41497 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:26:47,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:26:47,199 INFO  [M:0;d9f49988d155:41497 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:26:47,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41497-0x1019a3115bd0000, quorum=127.0.0.1:61029, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:26:47,202 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5381625b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:26:47,202 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22fd1e24{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:26:47,202 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:26:47,202 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6bd1231f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:26:47,202 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8afa355{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,STOPPED}
2024-12-10T02:26:47,203 WARN  [BP-320424626-172.17.0.2-1733797554327 heartbeating to localhost/127.0.0.1:44473 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:26:47,203 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:26:47,204 WARN  [BP-320424626-172.17.0.2-1733797554327 heartbeating to localhost/127.0.0.1:44473 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-320424626-172.17.0.2-1733797554327 (Datanode Uuid 29c68f3d-9319-4414-81d4-8f8f8d5c9a91) service to localhost/127.0.0.1:44473
2024-12-10T02:26:47,204 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:26:47,204 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data3/current/BP-320424626-172.17.0.2-1733797554327 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:26:47,204 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data4/current/BP-320424626-172.17.0.2-1733797554327 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:26:47,205 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:26:47,206 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3fdc15a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:26:47,206 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8d4c846{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:26:47,207 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:26:47,207 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78617008{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:26:47,207 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f08894b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,STOPPED}
2024-12-10T02:26:47,208 WARN  [BP-320424626-172.17.0.2-1733797554327 heartbeating to localhost/127.0.0.1:44473 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:26:47,208 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:26:47,208 WARN  [BP-320424626-172.17.0.2-1733797554327 heartbeating to localhost/127.0.0.1:44473 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-320424626-172.17.0.2-1733797554327 (Datanode Uuid 2921afbf-f93e-42b0-bed3-776ea216fbea) service to localhost/127.0.0.1:44473
2024-12-10T02:26:47,208 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:26:47,209 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data1/current/BP-320424626-172.17.0.2-1733797554327 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:26:47,209 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/cluster_d1d441a1-8386-ab17-9193-fbdfb187e204/data/data2/current/BP-320424626-172.17.0.2-1733797554327 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:26:47,209 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:26:47,216 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fa01b00{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:26:47,216 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@44b58721{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:26:47,216 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:26:47,216 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42501282{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:26:47,216 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dd613b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir/,STOPPED}
2024-12-10T02:26:47,222 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:26:47,239 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:26:47,249 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=210 (was 181)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44473 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: regionserver/d9f49988d155:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-10
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44473
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44473 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:44473
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44473
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44473
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44473 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-12
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-11
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44473
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=485 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=17 (was 36), ProcessCount=11 (was 11), AvailableMemoryMB=3721 (was 3764)
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=210, OpenFileDescriptor=485, MaxFileDescriptor=1048576, SystemLoadAverage=17, ProcessCount=11, AvailableMemoryMB=3721
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.log.dir so I do NOT create it in target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3f4821e2-4f15-c6be-dfc6-414a445baa36/hadoop.tmp.dir so I do NOT create it in target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7, deleteOnExit=true
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/test.cache.data in system properties and HBase conf
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:26:47,257 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:26:47,258 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:26:47,258 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:26:47,259 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:26:47,271 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:26:47,309 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:26:47,330 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:26:47,334 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:26:47,335 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:26:47,335 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:26:47,335 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:26:47,335 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:26:47,336 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b7f954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:26:47,336 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@331b1dff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:26:47,449 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a48d3d4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/java.io.tmpdir/jetty-localhost-42025-hadoop-hdfs-3_4_1-tests_jar-_-any-3292592280215874855/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:26:47,450 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0ef025{HTTP/1.1, (http/1.1)}{localhost:42025}
2024-12-10T02:26:47,450 INFO  [Time-limited test {}] server.Server(415): Started @236508ms
2024-12-10T02:26:47,462 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:26:47,564 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:26:47,569 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:26:47,569 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:26:47,569 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:26:47,570 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:26:47,570 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75b3fca0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:26:47,570 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@26d44036{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:26:47,698 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@425d5d71{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/java.io.tmpdir/jetty-localhost-39461-hadoop-hdfs-3_4_1-tests_jar-_-any-17804100946068415393/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:26:47,698 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4881a2ed{HTTP/1.1, (http/1.1)}{localhost:39461}
2024-12-10T02:26:47,699 INFO  [Time-limited test {}] server.Server(415): Started @236757ms
2024-12-10T02:26:47,700 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:26:47,727 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:26:47,730 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:26:47,731 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:26:47,731 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:26:47,731 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:26:47,731 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79ecb530{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:26:47,731 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@191b8d86{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:26:47,804 WARN  [Thread-1958 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data2/current/BP-1725285643-172.17.0.2-1733797607278/current, will proceed with Du for space computation calculation, 
2024-12-10T02:26:47,804 WARN  [Thread-1957 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data1/current/BP-1725285643-172.17.0.2-1733797607278/current, will proceed with Du for space computation calculation, 
2024-12-10T02:26:47,823 WARN  [Thread-1936 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:26:47,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0918b683ed846b3 with lease ID 0xe63f4982af11abaa: Processing first storage report for DS-c99b7d37-1ab8-4f1f-a7fb-c8c7d9643073 from datanode DatanodeRegistration(127.0.0.1:43385, datanodeUuid=3e41693d-c85b-4b7a-86ed-cbf218d21a1f, infoPort=42801, infoSecurePort=0, ipcPort=46301, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278)
2024-12-10T02:26:47,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0918b683ed846b3 with lease ID 0xe63f4982af11abaa: from storage DS-c99b7d37-1ab8-4f1f-a7fb-c8c7d9643073 node DatanodeRegistration(127.0.0.1:43385, datanodeUuid=3e41693d-c85b-4b7a-86ed-cbf218d21a1f, infoPort=42801, infoSecurePort=0, ipcPort=46301, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:26:47,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0918b683ed846b3 with lease ID 0xe63f4982af11abaa: Processing first storage report for DS-e5d309fe-f9b2-431c-88b6-dde011a13ddd from datanode DatanodeRegistration(127.0.0.1:43385, datanodeUuid=3e41693d-c85b-4b7a-86ed-cbf218d21a1f, infoPort=42801, infoSecurePort=0, ipcPort=46301, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278)
2024-12-10T02:26:47,825 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0918b683ed846b3 with lease ID 0xe63f4982af11abaa: from storage DS-e5d309fe-f9b2-431c-88b6-dde011a13ddd node DatanodeRegistration(127.0.0.1:43385, datanodeUuid=3e41693d-c85b-4b7a-86ed-cbf218d21a1f, infoPort=42801, infoSecurePort=0, ipcPort=46301, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:26:47,849 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5965a901{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/java.io.tmpdir/jetty-localhost-35205-hadoop-hdfs-3_4_1-tests_jar-_-any-5078137214379883330/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:26:47,849 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6f87fe6{HTTP/1.1, (http/1.1)}{localhost:35205}
2024-12-10T02:26:47,849 INFO  [Time-limited test {}] server.Server(415): Started @236908ms
2024-12-10T02:26:47,851 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:26:47,941 WARN  [Thread-1983 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data3/current/BP-1725285643-172.17.0.2-1733797607278/current, will proceed with Du for space computation calculation, 
2024-12-10T02:26:47,941 WARN  [Thread-1984 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data4/current/BP-1725285643-172.17.0.2-1733797607278/current, will proceed with Du for space computation calculation, 
2024-12-10T02:26:47,957 WARN  [Thread-1972 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:26:47,959 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad20802764655431 with lease ID 0xe63f4982af11abab: Processing first storage report for DS-4b4a1b97-5b63-4762-91c4-5af797943197 from datanode DatanodeRegistration(127.0.0.1:32957, datanodeUuid=7c1276fa-76aa-425d-807a-7d3259af4183, infoPort=35679, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278)
2024-12-10T02:26:47,959 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad20802764655431 with lease ID 0xe63f4982af11abab: from storage DS-4b4a1b97-5b63-4762-91c4-5af797943197 node DatanodeRegistration(127.0.0.1:32957, datanodeUuid=7c1276fa-76aa-425d-807a-7d3259af4183, infoPort=35679, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:26:47,960 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad20802764655431 with lease ID 0xe63f4982af11abab: Processing first storage report for DS-4bd97727-cee5-4429-87b2-52ba8a6c7921 from datanode DatanodeRegistration(127.0.0.1:32957, datanodeUuid=7c1276fa-76aa-425d-807a-7d3259af4183, infoPort=35679, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278)
2024-12-10T02:26:47,960 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad20802764655431 with lease ID 0xe63f4982af11abab: from storage DS-4bd97727-cee5-4429-87b2-52ba8a6c7921 node DatanodeRegistration(127.0.0.1:32957, datanodeUuid=7c1276fa-76aa-425d-807a-7d3259af4183, infoPort=35679, infoSecurePort=0, ipcPort=42429, storageInfo=lv=-57;cid=testClusterID;nsid=805170180;c=1733797607278), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T02:26:47,972 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd
2024-12-10T02:26:47,975 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/zookeeper_0, clientPort=52049, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:26:47,976 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52049
2024-12-10T02:26:47,976 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:47,977 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:47,985 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:26:47,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:26:47,987 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e with version=8
2024-12-10T02:26:47,987 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:26:47,988 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:26:47,989 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46239
2024-12-10T02:26:47,991 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46239 connecting to ZooKeeper ensemble=127.0.0.1:52049
2024-12-10T02:26:47,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462390x0, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:26:47,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46239-0x1019a31e4b50000 connected
2024-12-10T02:26:48,016 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,017 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,020 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:26:48,020 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e, hbase.cluster.distributed=false
2024-12-10T02:26:48,021 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:26:48,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46239
2024-12-10T02:26:48,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46239
2024-12-10T02:26:48,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46239
2024-12-10T02:26:48,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46239
2024-12-10T02:26:48,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46239
2024-12-10T02:26:48,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:26:48,042 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:26:48,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:48,043 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36643
2024-12-10T02:26:48,044 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36643 connecting to ZooKeeper ensemble=127.0.0.1:52049
2024-12-10T02:26:48,044 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,046 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366430x0, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:26:48,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366430x0, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:26:48,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36643-0x1019a31e4b50001 connected
2024-12-10T02:26:48,052 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:26:48,053 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:26:48,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:26:48,054 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:26:48,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36643
2024-12-10T02:26:48,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36643
2024-12-10T02:26:48,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36643
2024-12-10T02:26:48,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36643
2024-12-10T02:26:48,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36643
2024-12-10T02:26:48,071 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:46239
2024-12-10T02:26:48,071 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:26:48,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:26:48,074 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:26:48,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,075 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:26:48,076 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,46239,1733797607988 from backup master directory
2024-12-10T02:26:48,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:26:48,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:26:48,077 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:26:48,077 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,081 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/hbase.id] with ID: f6b96479-d59d-4a34-bf7f-530485ba7029
2024-12-10T02:26:48,081 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/.tmp/hbase.id
2024-12-10T02:26:48,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:26:48,087 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:26:48,087 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/.tmp/hbase.id]:[hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/hbase.id]
2024-12-10T02:26:48,097 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,097 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:26:48,099 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-10T02:26:48,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,106 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:26:48,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:26:48,107 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:26:48,108 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:26:48,108 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:26:48,114 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:26:48,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:26:48,116 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store
2024-12-10T02:26:48,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:26:48,121 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:26:48,121 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:26:48,122 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:48,122 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797608122Disabling compacts and flushes for region at 1733797608122Disabling writes for close at 1733797608122Writing region close event to WAL at 1733797608122Closed at 1733797608122
2024-12-10T02:26:48,122 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/.initializing
2024-12-10T02:26:48,122 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/WALs/d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,125 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C46239%2C1733797607988, suffix=, logDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/WALs/d9f49988d155,46239,1733797607988, archiveDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/oldWALs, maxLogs=10
2024-12-10T02:26:48,125 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C46239%2C1733797607988.1733797608125
2024-12-10T02:26:48,129 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/WALs/d9f49988d155,46239,1733797607988/d9f49988d155%2C46239%2C1733797607988.1733797608125
2024-12-10T02:26:48,130 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35679:35679),(127.0.0.1/127.0.0.1:42801:42801)]
2024-12-10T02:26:48,130 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:26:48,131 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:48,131 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,131 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,132 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:26:48,133 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:26:48,133 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,133 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,134 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:26:48,134 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:26:48,135 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,135 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:26:48,135 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:26:48,136 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:26:48,136 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,136 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:26:48,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:26:48,137 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:26:48,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,138 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:26:48,138 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,139 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,139 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,140 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:26:48,141 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:26:48,143 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:26:48,144 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=725535, jitterRate=-0.07743576169013977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:26:48,144 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797608131Initializing all the Stores at 1733797608131Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608131Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797608132 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797608132Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797608132Cleaning up temporary data from old regions at 1733797608140 (+8 ms)Region opened successfully at 1733797608144 (+4 ms)
2024-12-10T02:26:48,144 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:26:48,147 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3bab94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:26:48,148 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:26:48,148 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:26:48,148 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:26:48,148 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:26:48,149 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:26:48,149 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:26:48,149 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:26:48,151 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:26:48,151 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:26:48,152 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:26:48,153 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:26:48,153 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:26:48,154 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:26:48,154 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:26:48,156 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:26:48,157 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:26:48,158 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:26:48,161 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:26:48,162 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:26:48,163 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:26:48,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:26:48,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:26:48,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,165 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,46239,1733797607988, sessionid=0x1019a31e4b50000, setting cluster-up flag (Was=false)
2024-12-10T02:26:48,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,173 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:26:48,174 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,188 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:26:48,189 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,189 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:26:48,191 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:26:48,191 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:26:48,191 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:26:48,191 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,46239,1733797607988
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:26:48,192 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:26:48,192 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:26:48,192 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:26:48,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:26:48,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:26:48,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:26:48,193 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797638193
2024-12-10T02:26:48,193 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:26:48,194 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:26:48,194 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:26:48,194 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:26:48,195 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797608195,5,FailOnTimeoutGroup]
2024-12-10T02:26:48,195 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797608195,5,FailOnTimeoutGroup]
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,195 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,195 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,195 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:26:48,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:26:48,201 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:26:48,202 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:26:48,202 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e
2024-12-10T02:26:48,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:26:48,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:26:48,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:48,215 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:26:48,217 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:26:48,217 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,217 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:26:48,218 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:26:48,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,219 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,219 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:26:48,220 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:26:48,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,220 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,220 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:26:48,221 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:26:48,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,222 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:26:48,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740
2024-12-10T02:26:48,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740
2024-12-10T02:26:48,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:26:48,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:26:48,224 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:26:48,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:26:48,227 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:26:48,227 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=742619, jitterRate=-0.05571115016937256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797608214Initializing all the Stores at 1733797608215 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608215Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608215Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797608215Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608215Cleaning up temporary data from old regions at 1733797608224 (+9 ms)Region opened successfully at 1733797608228 (+4 ms)
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:26:48,228 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:26:48,228 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:26:48,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797608228Disabling compacts and flushes for region at 1733797608228Disabling writes for close at 1733797608228Writing region close event to WAL at 1733797608228Closed at 1733797608228
2024-12-10T02:26:48,230 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:26:48,230 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:26:48,230 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:26:48,231 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:26:48,232 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:26:48,261 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(746): ClusterId : f6b96479-d59d-4a34-bf7f-530485ba7029
2024-12-10T02:26:48,261 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:26:48,263 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:26:48,263 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:26:48,265 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:26:48,265 DEBUG [RS:0;d9f49988d155:36643 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7554d778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:26:48,277 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:36643
2024-12-10T02:26:48,277 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:26:48,277 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:26:48,277 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:26:48,277 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,46239,1733797607988 with port=36643, startcode=1733797608041
2024-12-10T02:26:48,278 DEBUG [RS:0;d9f49988d155:36643 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:26:48,279 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50465, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:26:48,280 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,280 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,281 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e
2024-12-10T02:26:48,281 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42629
2024-12-10T02:26:48,282 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:26:48,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:26:48,283 DEBUG [RS:0;d9f49988d155:36643 {}] zookeeper.ZKUtil(111): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,284 WARN  [RS:0;d9f49988d155:36643 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:26:48,284 INFO  [RS:0;d9f49988d155:36643 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:26:48,284 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,284 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,36643,1733797608041]
2024-12-10T02:26:48,287 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:26:48,288 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:26:48,289 INFO  [RS:0;d9f49988d155:36643 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:26:48,289 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,290 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:26:48,291 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:26:48,291 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:26:48,291 DEBUG [RS:0;d9f49988d155:36643 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,292 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36643,1733797608041-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:26:48,307 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:26:48,307 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,36643,1733797608041-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,307 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,307 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.Replication(171): d9f49988d155,36643,1733797608041 started
2024-12-10T02:26:48,321 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,321 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,36643,1733797608041, RpcServer on d9f49988d155/172.17.0.2:36643, sessionid=0x1019a31e4b50001
2024-12-10T02:26:48,321 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:26:48,321 DEBUG [RS:0;d9f49988d155:36643 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,321 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,36643,1733797608041'
2024-12-10T02:26:48,321 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,36643,1733797608041'
2024-12-10T02:26:48,322 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:26:48,323 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:26:48,323 DEBUG [RS:0;d9f49988d155:36643 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:26:48,323 INFO  [RS:0;d9f49988d155:36643 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:26:48,323 INFO  [RS:0;d9f49988d155:36643 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:26:48,382 WARN  [d9f49988d155:46239 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:26:48,425 INFO  [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C36643%2C1733797608041, suffix=, logDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041, archiveDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs, maxLogs=32
2024-12-10T02:26:48,425 INFO  [RS:0;d9f49988d155:36643 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C36643%2C1733797608041.1733797608425
2024-12-10T02:26:48,431 INFO  [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797608425
2024-12-10T02:26:48,431 DEBUG [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42801:42801),(127.0.0.1/127.0.0.1:35679:35679)]
2024-12-10T02:26:48,632 DEBUG [d9f49988d155:46239 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:26:48,633 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,634 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,36643,1733797608041, state=OPENING
2024-12-10T02:26:48,637 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:26:48,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:26:48,639 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:26:48,639 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:26:48,639 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,36643,1733797608041}]
2024-12-10T02:26:48,639 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:26:48,792 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:26:48,794 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49451, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:26:48,797 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:26:48,797 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:26:48,799 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C36643%2C1733797608041.meta, suffix=.meta, logDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041, archiveDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs, maxLogs=32
2024-12-10T02:26:48,800 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C36643%2C1733797608041.meta.1733797608799.meta
2024-12-10T02:26:48,805 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.meta.1733797608799.meta
2024-12-10T02:26:48,806 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42801:42801),(127.0.0.1/127.0.0.1:35679:35679)]
2024-12-10T02:26:48,806 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:26:48,807 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:26:48,807 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:26:48,808 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:26:48,809 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:26:48,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,810 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:26:48,811 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:26:48,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,811 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,812 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:26:48,812 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:26:48,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,813 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:26:48,813 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:26:48,813 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,814 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:26:48,814 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:26:48,814 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740
2024-12-10T02:26:48,815 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740
2024-12-10T02:26:48,817 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:26:48,817 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:26:48,817 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:26:48,818 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:26:48,819 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707237, jitterRate=-0.10070237517356873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:26:48,819 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:26:48,819 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797608807Writing region info on filesystem at 1733797608807Initializing all the Stores at 1733797608808 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608808Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608808Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797608808Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797608808Cleaning up temporary data from old regions at 1733797608817 (+9 ms)Running coprocessor post-open hooks at 1733797608819 (+2 ms)Region opened successfully at 1733797608819
2024-12-10T02:26:48,820 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797608792
2024-12-10T02:26:48,822 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:26:48,822 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:26:48,823 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,824 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,36643,1733797608041, state=OPEN
2024-12-10T02:26:48,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:26:48,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:26:48,828 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,36643,1733797608041
2024-12-10T02:26:48,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:26:48,828 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:26:48,830 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:26:48,830 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,36643,1733797608041 in 189 msec
2024-12-10T02:26:48,833 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:26:48,833 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 600 msec
2024-12-10T02:26:48,833 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:26:48,834 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:26:48,835 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:26:48,835 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,36643,1733797608041, seqNum=-1]
2024-12-10T02:26:48,835 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:26:48,836 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39101, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:26:48,841 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 649 msec
2024-12-10T02:26:48,842 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797608841, completionTime=-1
2024-12-10T02:26:48,842 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:26:48,842 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:26:48,843 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:26:48,843 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797668843
2024-12-10T02:26:48,843 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797728843
2024-12-10T02:26:48,843 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:46239, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,844 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,846 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.771sec
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:26:48,848 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:26:48,850 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:26:48,850 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:26:48,851 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,46239,1733797607988-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:26:48,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d29260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:26:48,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,46239,-1 for getting cluster id
2024-12-10T02:26:48,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:26:48,862 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6b96479-d59d-4a34-bf7f-530485ba7029'
2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6b96479-d59d-4a34-bf7f-530485ba7029"

2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6eae0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,46239,-1]
2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:26:48,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:26:48,864 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:26:48,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f940032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:26:48,865 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:26:48,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,36643,1733797608041, seqNum=-1]
2024-12-10T02:26:48,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:26:48,867 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:26:48,869 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,869 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:26:48,871 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:26:48,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-10T02:26:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is d9f49988d155,46239,1733797607988
2024-12-10T02:26:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58987a54
2024-12-10T02:26:48,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-10T02:26:48,873 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-10T02:26:48,873 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-12-10T02:26:48,874 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-12-10T02:26:48,874 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:26:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-12-10T02:26:48,876 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-12-10T02:26:48,876 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:48,876 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-12-10T02:26:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:26:48,877 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-10T02:26:48,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741835_1011 (size=381)
2024-12-10T02:26:48,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741835_1011 (size=381)
2024-12-10T02:26:48,886 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8650122bf6158d757da060b6cb41f4fe, NAME => 'TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e
2024-12-10T02:26:48,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741836_1012 (size=64)
2024-12-10T02:26:48,891 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741836_1012 (size=64)
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 8650122bf6158d757da060b6cb41f4fe, disabling compactions & flushes
2024-12-10T02:26:48,892 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. after waiting 0 ms
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:48,892 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:48,892 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8650122bf6158d757da060b6cb41f4fe:
Waiting for close lock at 1733797608892Disabling compacts and flushes for region at 1733797608892Disabling writes for close at 1733797608892Writing region close event to WAL at 1733797608892Closed at 1733797608892
2024-12-10T02:26:48,893 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-12-10T02:26:48,894 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733797608893"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797608893"}]},"ts":"1733797608893"}
2024-12-10T02:26:48,896 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-10T02:26:48,897 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-10T02:26:48,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797608897"}]},"ts":"1733797608897"}
2024-12-10T02:26:48,899 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-12-10T02:26:48,899 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, ASSIGN}]
2024-12-10T02:26:48,901 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, ASSIGN
2024-12-10T02:26:48,902 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, ASSIGN; state=OFFLINE, location=d9f49988d155,36643,1733797608041; forceNewPlan=false, retain=false
2024-12-10T02:26:49,040 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:49,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:49,052 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8650122bf6158d757da060b6cb41f4fe, regionState=OPENING, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:26:49,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, ASSIGN because future has completed
2024-12-10T02:26:49,055 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041}]
2024-12-10T02:26:49,211 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:49,211 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8650122bf6158d757da060b6cb41f4fe, NAME => 'TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:26:49,211 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,212 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:26:49,212 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,212 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,213 INFO  [StoreOpener-8650122bf6158d757da060b6cb41f4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8650122bf6158d757da060b6cb41f4fe 
2024-12-10T02:26:49,214 INFO  [StoreOpener-8650122bf6158d757da060b6cb41f4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8650122bf6158d757da060b6cb41f4fe columnFamilyName info
2024-12-10T02:26:49,214 DEBUG [StoreOpener-8650122bf6158d757da060b6cb41f4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:26:49,215 INFO  [StoreOpener-8650122bf6158d757da060b6cb41f4fe-1 {}] regionserver.HStore(327): Store=8650122bf6158d757da060b6cb41f4fe/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:26:49,215 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,215 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,215 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,216 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,216 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,217 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,219 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:26:49,219 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8650122bf6158d757da060b6cb41f4fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752461, jitterRate=-0.04319754242897034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:26:49,219 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:49,220 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8650122bf6158d757da060b6cb41f4fe:
Running coprocessor pre-open hook at 1733797609212Writing region info on filesystem at 1733797609212Initializing all the Stores at 1733797609212Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797609212Cleaning up temporary data from old regions at 1733797609216 (+4 ms)Running coprocessor post-open hooks at 1733797609219 (+3 ms)Region opened successfully at 1733797609220 (+1 ms)
2024-12-10T02:26:49,221 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., pid=6, masterSystemTime=1733797609207
2024-12-10T02:26:49,223 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:49,223 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:49,223 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8650122bf6158d757da060b6cb41f4fe, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:26:49,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041 because future has completed
2024-12-10T02:26:49,229 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-10T02:26:49,229 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041 in 171 msec
2024-12-10T02:26:49,231 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-10T02:26:49,231 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, ASSIGN in 330 msec
2024-12-10T02:26:49,232 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-10T02:26:49,232 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733797609232"}]},"ts":"1733797609232"}
2024-12-10T02:26:49,234 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-12-10T02:26:49,235 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-12-10T02:26:49,236 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 361 msec
2024-12-10T02:26:50,041 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:50,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:51,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:51,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:51,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,786 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,787 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,812 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,812 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,813 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,814 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:51,822 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,042 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:52,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:52,328 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:26:52,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,329 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,330 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,356 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,357 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,357 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,358 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,363 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,364 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:52,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:26:53,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:53,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:54,043 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:54,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:54,287 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:26:54,288 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-12-10T02:26:55,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:55,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:56,044 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:56,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:56,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-10T02:26:56,235 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-12-10T02:26:56,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-12-10T02:26:57,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:57,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:58,045 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:58,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-10T02:26:58,974 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-12-10T02:26:58,975 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-12-10T02:26:58,977 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-12-10T02:26:58,977 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:26:58,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2]
2024-12-10T02:26:58,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:26:58,991 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:26:59,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/09490c079ebf49b7b3fae1a1309992a1 is 1080, key is row0001/info:/1733797618981/Put/seqid=0
2024-12-10T02:26:59,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741837_1013 (size=12509)
2024-12-10T02:26:59,016 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741837_1013 (size=12509)
2024-12-10T02:26:59,016 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/09490c079ebf49b7b3fae1a1309992a1
2024-12-10T02:26:59,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/09490c079ebf49b7b3fae1a1309992a1 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1
2024-12-10T02:26:59,029 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1, entries=7, sequenceid=11, filesize=12.2 K
2024-12-10T02:26:59,029 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-10T02:26:59,030 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 8650122bf6158d757da060b6cb41f4fe in 39ms, sequenceid=11, compaction requested=false
2024-12-10T02:26:59,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:26:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:43062 deadline: 1733797629028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
2024-12-10T02:26:59,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:59,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:26:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:26:59,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:26:59,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 because the exception is null or not the one we care about
2024-12-10T02:27:00,046 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:00,047 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:01,047 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:01,047 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:01,737 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:27:01,738 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,738 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,739 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,739 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,739 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,739 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,760 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,761 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,764 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:01,767 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:02,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:02,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:03,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:03,048 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:04,049 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:04,049 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:05,049 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:05,050 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:06,050 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:06,050 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:07,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:07,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:08,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:08,051 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:09,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:09,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:09,156 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-12-10T02:27:09,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/c2b6ac15940443c097f12bf3ebefa45a is 1080, key is row0008/info:/1733797618992/Put/seqid=0
2024-12-10T02:27:09,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741838_1014 (size=29761)
2024-12-10T02:27:09,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741838_1014 (size=29761)
2024-12-10T02:27:09,167 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/c2b6ac15940443c097f12bf3ebefa45a
2024-12-10T02:27:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/c2b6ac15940443c097f12bf3ebefa45a as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a
2024-12-10T02:27:09,177 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a, entries=23, sequenceid=37, filesize=29.1 K
2024-12-10T02:27:09,178 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 8650122bf6158d757da060b6cb41f4fe in 22ms, sequenceid=37, compaction requested=false
2024-12-10T02:27:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-12-10T02:27:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:09,178 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a because midkey is the same as first or last row
2024-12-10T02:27:10,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:10,052 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:11,053 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:11,053 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:11,170 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:27:11,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/4e4613beb5644c499ab6e040685036bd is 1080, key is row0031/info:/1733797629158/Put/seqid=0
2024-12-10T02:27:11,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741839_1015 (size=12509)
2024-12-10T02:27:11,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741839_1015 (size=12509)
2024-12-10T02:27:11,181 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/4e4613beb5644c499ab6e040685036bd
2024-12-10T02:27:11,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/4e4613beb5644c499ab6e040685036bd as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd
2024-12-10T02:27:11,193 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd, entries=7, sequenceid=47, filesize=12.2 K
2024-12-10T02:27:11,194 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 8650122bf6158d757da060b6cb41f4fe in 24ms, sequenceid=47, compaction requested=true
2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a because midkey is the same as first or last row
2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8650122bf6158d757da060b6cb41f4fe:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:11,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:11,194 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:11,195 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T02:27:11,196 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:11,196 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 8650122bf6158d757da060b6cb41f4fe/info is initiating minor compaction (all files)
2024-12-10T02:27:11,196 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8650122bf6158d757da060b6cb41f4fe/info in TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:11,196 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp, totalSize=53.5 K
2024-12-10T02:27:11,196 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09490c079ebf49b7b3fae1a1309992a1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733797618981
2024-12-10T02:27:11,197 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2b6ac15940443c097f12bf3ebefa45a, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1733797618992
2024-12-10T02:27:11,197 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4613beb5644c499ab6e040685036bd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733797629158
2024-12-10T02:27:11,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/eea7bb3ebaaa43cd8c915720b64dcc58 is 1080, key is row0038/info:/1733797631171/Put/seqid=0
2024-12-10T02:27:11,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741840_1016 (size=20064)
2024-12-10T02:27:11,205 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741840_1016 (size=20064)
2024-12-10T02:27:11,205 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/eea7bb3ebaaa43cd8c915720b64dcc58
2024-12-10T02:27:11,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/eea7bb3ebaaa43cd8c915720b64dcc58 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58
2024-12-10T02:27:11,217 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8650122bf6158d757da060b6cb41f4fe#info#compaction#59 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:11,218 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/602ef7165d2a4181af5377dfde668b33 is 1080, key is row0001/info:/1733797618981/Put/seqid=0
2024-12-10T02:27:11,220 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58, entries=14, sequenceid=64, filesize=19.6 K
2024-12-10T02:27:11,221 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 8650122bf6158d757da060b6cb41f4fe in 26ms, sequenceid=64, compaction requested=false
2024-12-10T02:27:11,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:11,221 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.1 K, sizeToCheck=16.0 K
2024-12-10T02:27:11,221 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:11,221 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a because midkey is the same as first or last row
2024-12-10T02:27:11,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741841_1017 (size=44978)
2024-12-10T02:27:11,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741841_1017 (size=44978)
2024-12-10T02:27:11,232 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/602ef7165d2a4181af5377dfde668b33 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33
2024-12-10T02:27:11,239 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8650122bf6158d757da060b6cb41f4fe/info of 8650122bf6158d757da060b6cb41f4fe into 602ef7165d2a4181af5377dfde668b33(size=43.9 K), total size for store is 63.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:11,239 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., storeName=8650122bf6158d757da060b6cb41f4fe/info, priority=13, startTime=1733797631194; duration=0sec
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 because midkey is the same as first or last row
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 because midkey is the same as first or last row
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 because midkey is the same as first or last row
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:11,239 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8650122bf6158d757da060b6cb41f4fe:info
2024-12-10T02:27:12,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:12,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:13,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:13,054 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,222 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T02:27:13,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/a66e6c0d16f64ecf96bf79c894ecbae0 is 1080, key is row0052/info:/1733797631196/Put/seqid=0
2024-12-10T02:27:13,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741842_1018 (size=20064)
2024-12-10T02:27:13,233 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741842_1018 (size=20064)
2024-12-10T02:27:13,234 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/a66e6c0d16f64ecf96bf79c894ecbae0
2024-12-10T02:27:13,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/a66e6c0d16f64ecf96bf79c894ecbae0 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0
2024-12-10T02:27:13,245 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0, entries=14, sequenceid=82, filesize=19.6 K
2024-12-10T02:27:13,246 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 8650122bf6158d757da060b6cb41f4fe in 24ms, sequenceid=82, compaction requested=true
2024-12-10T02:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K
2024-12-10T02:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 because midkey is the same as first or last row
2024-12-10T02:27:13,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8650122bf6158d757da060b6cb41f4fe:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:13,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,247 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,248 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:13,248 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T02:27:13,248 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 8650122bf6158d757da060b6cb41f4fe/info is initiating minor compaction (all files)
2024-12-10T02:27:13,248 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8650122bf6158d757da060b6cb41f4fe/info in TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:13,248 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp, totalSize=83.1 K
2024-12-10T02:27:13,249 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 602ef7165d2a4181af5377dfde668b33, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733797618981
2024-12-10T02:27:13,249 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting eea7bb3ebaaa43cd8c915720b64dcc58, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733797631171
2024-12-10T02:27:13,250 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66e6c0d16f64ecf96bf79c894ecbae0, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733797631196
2024-12-10T02:27:13,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/d71d0757e17d4c8598102d5a7b9e93f0 is 1080, key is row0066/info:/1733797633223/Put/seqid=0
2024-12-10T02:27:13,262 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741843_1019 (size=20064)
2024-12-10T02:27:13,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741843_1019 (size=20064)
2024-12-10T02:27:13,263 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/d71d0757e17d4c8598102d5a7b9e93f0
2024-12-10T02:27:13,265 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8650122bf6158d757da060b6cb41f4fe#info#compaction#62 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:13,265 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/90ae33ee017641dbacfddf352ac8bc56 is 1080, key is row0001/info:/1733797618981/Put/seqid=0
2024-12-10T02:27:13,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/d71d0757e17d4c8598102d5a7b9e93f0 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/d71d0757e17d4c8598102d5a7b9e93f0
2024-12-10T02:27:13,272 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741844_1020 (size=75378)
2024-12-10T02:27:13,273 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741844_1020 (size=75378)
2024-12-10T02:27:13,276 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-10T02:27:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:43062 deadline: 1733797643276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,277 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/d71d0757e17d4c8598102d5a7b9e93f0, entries=14, sequenceid=99, filesize=19.6 K
2024-12-10T02:27:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:13,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 because the exception is null or not the one we care about
2024-12-10T02:27:13,278 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=15.76 KB/16140 for 8650122bf6158d757da060b6cb41f4fe in 30ms, sequenceid=99, compaction requested=false
2024-12-10T02:27:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.7 K, sizeToCheck=16.0 K
2024-12-10T02:27:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:13,278 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 because midkey is the same as first or last row
2024-12-10T02:27:13,279 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/90ae33ee017641dbacfddf352ac8bc56 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56
2024-12-10T02:27:13,285 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8650122bf6158d757da060b6cb41f4fe/info of 8650122bf6158d757da060b6cb41f4fe into 90ae33ee017641dbacfddf352ac8bc56(size=73.6 K), total size for store is 93.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8650122bf6158d757da060b6cb41f4fe:

2024-12-10T02:27:13,285 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., storeName=8650122bf6158d757da060b6cb41f4fe/info, priority=13, startTime=1733797633246; duration=0sec
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K
2024-12-10T02:27:13,285 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-12-10T02:27:13,286 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,286 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,286 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8650122bf6158d757da060b6cb41f4fe:info
2024-12-10T02:27:13,287 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] assignment.AssignmentManager(1363): Split request from d9f49988d155,36643,1733797608041, parent={ENCODED => 8650122bf6158d757da060b6cb41f4fe, NAME => 'TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-12-10T02:27:13,292 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,296 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8650122bf6158d757da060b6cb41f4fe, daughterA=9d68098034d65e6395ee8af43e3246e7, daughterB=9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,297 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8650122bf6158d757da060b6cb41f4fe, daughterA=9d68098034d65e6395ee8af43e3246e7, daughterB=9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,297 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8650122bf6158d757da060b6cb41f4fe, daughterA=9d68098034d65e6395ee8af43e3246e7, daughterB=9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,297 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8650122bf6158d757da060b6cb41f4fe, daughterA=9d68098034d65e6395ee8af43e3246e7, daughterB=9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,303 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, UNASSIGN}]
2024-12-10T02:27:13,304 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, UNASSIGN
2024-12-10T02:27:13,305 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8650122bf6158d757da060b6cb41f4fe, regionState=CLOSING, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,307 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, UNASSIGN because future has completed
2024-12-10T02:27:13,308 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-12-10T02:27:13,308 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041}]
2024-12-10T02:27:13,464 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,464 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-12-10T02:27:13,465 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 8650122bf6158d757da060b6cb41f4fe, disabling compactions & flushes
2024-12-10T02:27:13,465 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:13,465 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:13,465 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. after waiting 0 ms
2024-12-10T02:27:13,465 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:13,465 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 8650122bf6158d757da060b6cb41f4fe 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-10T02:27:13,470 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/648cb9d596cc447caf3ed23fb57c22e1 is 1080, key is row0080/info:/1733797633249/Put/seqid=0
2024-12-10T02:27:13,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741845_1021 (size=21141)
2024-12-10T02:27:13,474 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741845_1021 (size=21141)
2024-12-10T02:27:13,475 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/648cb9d596cc447caf3ed23fb57c22e1
2024-12-10T02:27:13,480 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/.tmp/info/648cb9d596cc447caf3ed23fb57c22e1 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/648cb9d596cc447caf3ed23fb57c22e1
2024-12-10T02:27:13,485 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/648cb9d596cc447caf3ed23fb57c22e1, entries=15, sequenceid=118, filesize=20.6 K
2024-12-10T02:27:13,486 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for 8650122bf6158d757da060b6cb41f4fe in 21ms, sequenceid=118, compaction requested=true
2024-12-10T02:27:13,487 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0] to archive
2024-12-10T02:27:13,488 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:27:13,490 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/09490c079ebf49b7b3fae1a1309992a1
2024-12-10T02:27:13,490 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/c2b6ac15940443c097f12bf3ebefa45a
2024-12-10T02:27:13,491 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/602ef7165d2a4181af5377dfde668b33
2024-12-10T02:27:13,491 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/4e4613beb5644c499ab6e040685036bd
2024-12-10T02:27:13,491 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/a66e6c0d16f64ecf96bf79c894ecbae0
2024-12-10T02:27:13,491 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/eea7bb3ebaaa43cd8c915720b64dcc58
2024-12-10T02:27:13,497 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=1
2024-12-10T02:27:13,497 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.
2024-12-10T02:27:13,497 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 8650122bf6158d757da060b6cb41f4fe:
Waiting for close lock at 1733797633465Running coprocessor pre-close hooks at 1733797633465Disabling compacts and flushes for region at 1733797633465Disabling writes for close at 1733797633465Obtaining lock to block concurrent updates at 1733797633465Preparing flush snapshotting stores in 8650122bf6158d757da060b6cb41f4fe at 1733797633465Finished memstore snapshotting TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., syncing WAL and waiting on mvcc, flushsize=dataSize=16140, getHeapSize=17520, getOffHeapSize=0, getCellsCount=15 at 1733797633465Flushing stores of TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. at 1733797633466 (+1 ms)Flushing 8650122bf6158d757da060b6cb41f4fe/info: creating writer at 1733797633466Flushing 8650122bf6158d757da060b6cb41f4fe/info: appending metadata at 1733797633469 (+3 ms)Flushing 8650122bf6158d757da060b6cb41f4fe/info: closing flushed file at 1733797633469Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dab702b: reopening flushed file at 1733797633480 (+11 ms)Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=0 B/0 for 8650122bf6158d757da060b6cb41f4fe in 21ms, sequenceid=118, compaction requested=true at 1733797633486 (+6 ms)Writing region close event to WAL at 1733797633493 (+7 ms)Running coprocessor post-close hooks at 1733797633497 (+4 ms)Closed at 1733797633497
2024-12-10T02:27:13,499 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,500 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8650122bf6158d757da060b6cb41f4fe, regionState=CLOSED
2024-12-10T02:27:13,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041 because future has completed
2024-12-10T02:27:13,505 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-10T02:27:13,505 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 8650122bf6158d757da060b6cb41f4fe, server=d9f49988d155,36643,1733797608041 in 195 msec
2024-12-10T02:27:13,507 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-10T02:27:13,507 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8650122bf6158d757da060b6cb41f4fe, UNASSIGN in 202 msec
2024-12-10T02:27:13,515 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:27:13,518 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=8650122bf6158d757da060b6cb41f4fe, threads=3
2024-12-10T02:27:13,520 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/648cb9d596cc447caf3ed23fb57c22e1 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,520 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/d71d0757e17d4c8598102d5a7b9e93f0 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,520 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,530 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/d71d0757e17d4c8598102d5a7b9e93f0, top=true
2024-12-10T02:27:13,530 DEBUG [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/648cb9d596cc447caf3ed23fb57c22e1, top=true
2024-12-10T02:27:13,539 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0 for child: 9703a4063d9cca5dbdf91fb53d597ddf, parent: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,539 INFO  [StoreFileSplitter-pool-0 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1 for child: 9703a4063d9cca5dbdf91fb53d597ddf, parent: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,539 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/d71d0757e17d4c8598102d5a7b9e93f0 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,539 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/648cb9d596cc447caf3ed23fb57c22e1 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741846_1022 (size=27)
2024-12-10T02:27:13,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741846_1022 (size=27)
2024-12-10T02:27:13,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741847_1023 (size=27)
2024-12-10T02:27:13,557 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741847_1023 (size=27)
2024-12-10T02:27:13,558 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56 for region: 8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:27:13,560 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 8650122bf6158d757da060b6cb41f4fe Daughter A: [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe] storefiles, Daughter B: [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0] storefiles.
2024-12-10T02:27:13,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741848_1024 (size=71)
2024-12-10T02:27:13,567 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741848_1024 (size=71)
2024-12-10T02:27:13,569 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:27:13,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741849_1025 (size=71)
2024-12-10T02:27:13,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741849_1025 (size=71)
2024-12-10T02:27:13,582 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:27:13,591 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1
2024-12-10T02:27:13,593 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1
2024-12-10T02:27:13,595 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1733797633595"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1733797633595"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1733797633595"}]},"ts":"1733797633595"}
2024-12-10T02:27:13,595 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733797633595"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797633595"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733797633595"}]},"ts":"1733797633595"}
2024-12-10T02:27:13,595 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1733797633595"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733797633595"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1733797633595"}]},"ts":"1733797633595"}
2024-12-10T02:27:13,613 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d68098034d65e6395ee8af43e3246e7, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9703a4063d9cca5dbdf91fb53d597ddf, ASSIGN}]
2024-12-10T02:27:13,614 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d68098034d65e6395ee8af43e3246e7, ASSIGN
2024-12-10T02:27:13,614 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9703a4063d9cca5dbdf91fb53d597ddf, ASSIGN
2024-12-10T02:27:13,615 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d68098034d65e6395ee8af43e3246e7, ASSIGN; state=SPLITTING_NEW, location=d9f49988d155,36643,1733797608041; forceNewPlan=false, retain=false
2024-12-10T02:27:13,615 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9703a4063d9cca5dbdf91fb53d597ddf, ASSIGN; state=SPLITTING_NEW, location=d9f49988d155,36643,1733797608041; forceNewPlan=false, retain=false
2024-12-10T02:27:13,765 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9d68098034d65e6395ee8af43e3246e7, regionState=OPENING, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,765 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9703a4063d9cca5dbdf91fb53d597ddf, regionState=OPENING, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d68098034d65e6395ee8af43e3246e7, ASSIGN because future has completed
2024-12-10T02:27:13,768 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d68098034d65e6395ee8af43e3246e7, server=d9f49988d155,36643,1733797608041}]
2024-12-10T02:27:13,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9703a4063d9cca5dbdf91fb53d597ddf, ASSIGN because future has completed
2024-12-10T02:27:13,770 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041}]
2024-12-10T02:27:13,924 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:27:13,924 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d68098034d65e6395ee8af43e3246e7, NAME => 'TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.', STARTKEY => '', ENDKEY => 'row0062'}
2024-12-10T02:27:13,924 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,925 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:27:13,925 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,925 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,926 INFO  [StoreOpener-9d68098034d65e6395ee8af43e3246e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9d68098034d65e6395ee8af43e3246e7 
2024-12-10T02:27:13,926 INFO  [StoreOpener-9d68098034d65e6395ee8af43e3246e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d68098034d65e6395ee8af43e3246e7 columnFamilyName info
2024-12-10T02:27:13,927 DEBUG [StoreOpener-9d68098034d65e6395ee8af43e3246e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:27:13,937 DEBUG [StoreOpener-9d68098034d65e6395ee8af43e3246e7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-bottom
2024-12-10T02:27:13,937 INFO  [StoreOpener-9d68098034d65e6395ee8af43e3246e7-1 {}] regionserver.HStore(327): Store=9d68098034d65e6395ee8af43e3246e7/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:27:13,937 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,938 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,939 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,940 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,940 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,941 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,942 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 9d68098034d65e6395ee8af43e3246e7; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=747897, jitterRate=-0.04900027811527252}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:27:13,942 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:27:13,942 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 9d68098034d65e6395ee8af43e3246e7:
Running coprocessor pre-open hook at 1733797633925Writing region info on filesystem at 1733797633925Initializing all the Stores at 1733797633925Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797633925Cleaning up temporary data from old regions at 1733797633940 (+15 ms)Running coprocessor post-open hooks at 1733797633942 (+2 ms)Region opened successfully at 1733797633942
2024-12-10T02:27:13,943 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7., pid=12, masterSystemTime=1733797633921
2024-12-10T02:27:13,944 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 9d68098034d65e6395ee8af43e3246e7:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:13,944 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-12-10T02:27:13,944 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,944 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:27:13,944 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9d68098034d65e6395ee8af43e3246e7/info is initiating minor compaction (all files)
2024-12-10T02:27:13,944 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d68098034d65e6395ee8af43e3246e7/info in TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:27:13,945 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-bottom] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/.tmp, totalSize=73.6 K
2024-12-10T02:27:13,945 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733797618981
2024-12-10T02:27:13,946 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:27:13,946 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:27:13,946 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:13,946 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 9703a4063d9cca5dbdf91fb53d597ddf, NAME => 'TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.', STARTKEY => 'row0062', ENDKEY => ''}
2024-12-10T02:27:13,947 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,947 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:27:13,947 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,947 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,947 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9d68098034d65e6395ee8af43e3246e7, regionState=OPEN, openSeqNum=122, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,948 INFO  [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9703a4063d9cca5dbdf91fb53d597ddf 
2024-12-10T02:27:13,949 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-12-10T02:27:13,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-12-10T02:27:13,949 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB
2024-12-10T02:27:13,949 INFO  [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9703a4063d9cca5dbdf91fb53d597ddf columnFamilyName info
2024-12-10T02:27:13,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d68098034d65e6395ee8af43e3246e7, server=d9f49988d155,36643,1733797608041 because future has completed
2024-12-10T02:27:13,949 DEBUG [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:27:13,954 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-12-10T02:27:13,954 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d68098034d65e6395ee8af43e3246e7, server=d9f49988d155,36643,1733797608041 in 183 msec
2024-12-10T02:27:13,956 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d68098034d65e6395ee8af43e3246e7, ASSIGN in 341 msec
2024-12-10T02:27:13,958 DEBUG [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-top
2024-12-10T02:27:13,963 DEBUG [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1
2024-12-10T02:27:13,966 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d68098034d65e6395ee8af43e3246e7#info#compaction#64 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:13,967 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/.tmp/info/5c33fa61c838469aa215b930987842da is 1080, key is row0001/info:/1733797618981/Put/seqid=0
2024-12-10T02:27:13,967 DEBUG [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0
2024-12-10T02:27:13,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/9d41c3dcfb52451cbdcd919734cc5b49 is 193, key is TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf./info:regioninfo/1733797633765/Put/seqid=0
2024-12-10T02:27:13,967 INFO  [StoreOpener-9703a4063d9cca5dbdf91fb53d597ddf-1 {}] regionserver.HStore(327): Store=9703a4063d9cca5dbdf91fb53d597ddf/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:27:13,967 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,968 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,969 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,970 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,970 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741850_1026 (size=70862)
2024-12-10T02:27:13,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741850_1026 (size=70862)
2024-12-10T02:27:13,972 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741851_1027 (size=9847)
2024-12-10T02:27:13,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741851_1027 (size=9847)
2024-12-10T02:27:13,974 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 9703a4063d9cca5dbdf91fb53d597ddf; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703470, jitterRate=-0.10549220442771912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-10T02:27:13,974 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:13,974 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/9d41c3dcfb52451cbdcd919734cc5b49
2024-12-10T02:27:13,974 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 9703a4063d9cca5dbdf91fb53d597ddf:
Running coprocessor pre-open hook at 1733797633947Writing region info on filesystem at 1733797633947Initializing all the Stores at 1733797633948 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797633948Cleaning up temporary data from old regions at 1733797633970 (+22 ms)Running coprocessor post-open hooks at 1733797633974 (+4 ms)Region opened successfully at 1733797633974
2024-12-10T02:27:13,975 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., pid=13, masterSystemTime=1733797633921
2024-12-10T02:27:13,975 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 2
2024-12-10T02:27:13,975 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,975 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:13,976 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:13,976 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:13,977 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:13,977 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-top, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=113.9 K
2024-12-10T02:27:13,977 DEBUG [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:13,977 INFO  [RS_OPEN_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:13,977 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] compactions.Compactor(225): Compacting 90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733797618981
2024-12-10T02:27:13,978 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733797633223
2024-12-10T02:27:13,978 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9703a4063d9cca5dbdf91fb53d597ddf, regionState=OPEN, openSeqNum=122, regionLocation=d9f49988d155,36643,1733797608041
2024-12-10T02:27:13,978 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733797633249
2024-12-10T02:27:13,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041 because future has completed
2024-12-10T02:27:13,989 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/.tmp/info/5c33fa61c838469aa215b930987842da as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/5c33fa61c838469aa215b930987842da
2024-12-10T02:27:13,990 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-12-10T02:27:13,990 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041 in 217 msec
2024-12-10T02:27:13,993 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7
2024-12-10T02:27:13,993 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9703a4063d9cca5dbdf91fb53d597ddf, ASSIGN in 377 msec
2024-12-10T02:27:13,996 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8650122bf6158d757da060b6cb41f4fe, daughterA=9d68098034d65e6395ee8af43e3246e7, daughterB=9703a4063d9cca5dbdf91fb53d597ddf in 701 msec
2024-12-10T02:27:13,997 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 9d68098034d65e6395ee8af43e3246e7/info of 9d68098034d65e6395ee8af43e3246e7 into 5c33fa61c838469aa215b930987842da(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:13,997 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d68098034d65e6395ee8af43e3246e7:

2024-12-10T02:27:13,997 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7., storeName=9d68098034d65e6395ee8af43e3246e7/info, priority=15, startTime=1733797633944; duration=0sec
2024-12-10T02:27:13,997 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:13,997 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d68098034d65e6395ee8af43e3246e7:info
2024-12-10T02:27:14,006 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#66 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:14,006 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/2f260b15a7504b4dac8ccc96b618baa8 is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:14,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/ns/e288a6fd4ce84972b9336ec3ff5e5686 is 43, key is default/ns:d/1733797608837/Put/seqid=0
2024-12-10T02:27:14,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741852_1028 (size=40830)
2024-12-10T02:27:14,012 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741852_1028 (size=40830)
2024-12-10T02:27:14,017 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/2f260b15a7504b4dac8ccc96b618baa8 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2f260b15a7504b4dac8ccc96b618baa8
2024-12-10T02:27:14,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741853_1029 (size=5153)
2024-12-10T02:27:14,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741853_1029 (size=5153)
2024-12-10T02:27:14,019 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/ns/e288a6fd4ce84972b9336ec3ff5e5686
2024-12-10T02:27:14,024 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into 2f260b15a7504b4dac8ccc96b618baa8(size=39.9 K), total size for store is 39.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:14,024 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:14,024 INFO  [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797633975; duration=0sec
2024-12-10T02:27:14,024 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:14,024 DEBUG [RS:0;d9f49988d155:36643-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:14,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/table/915991d36ea748a9b5f331190ad3295b is 65, key is TestLogRolling-testLogRolling/table:state/1733797609232/Put/seqid=0
2024-12-10T02:27:14,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741854_1030 (size=5340)
2024-12-10T02:27:14,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741854_1030 (size=5340)
2024-12-10T02:27:14,044 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/table/915991d36ea748a9b5f331190ad3295b
2024-12-10T02:27:14,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/9d41c3dcfb52451cbdcd919734cc5b49 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/info/9d41c3dcfb52451cbdcd919734cc5b49
2024-12-10T02:27:14,054 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/info/9d41c3dcfb52451cbdcd919734cc5b49, entries=30, sequenceid=17, filesize=9.6 K
2024-12-10T02:27:14,055 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:14,055 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:14,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/ns/e288a6fd4ce84972b9336ec3ff5e5686 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/ns/e288a6fd4ce84972b9336ec3ff5e5686
2024-12-10T02:27:14,060 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/ns/e288a6fd4ce84972b9336ec3ff5e5686, entries=2, sequenceid=17, filesize=5.0 K
2024-12-10T02:27:14,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/table/915991d36ea748a9b5f331190ad3295b as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/table/915991d36ea748a9b5f331190ad3295b
2024-12-10T02:27:14,065 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/table/915991d36ea748a9b5f331190ad3295b, entries=2, sequenceid=17, filesize=5.2 K
2024-12-10T02:27:14,067 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 117ms, sequenceid=17, compaction requested=false
2024-12-10T02:27:14,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T02:27:15,056 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:15,056 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:16,056 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:16,056 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:17,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:17,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:17,971 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:27:18,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:18,057 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:18,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,499 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,519 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,519 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,519 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,522 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:18,524 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,030 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:27:19,031 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,032 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,033 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,053 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,054 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,056 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,057 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,057 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:19,058 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:19,058 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:19,060 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:27:20,058 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:20,058 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:21,059 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:21,059 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:22,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:22,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:23,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:23,060 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:43062 deadline: 1733797653334, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. is not online on d9f49988d155,36643,1733797608041
2024-12-10T02:27:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. is not online on d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe. is not online on d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:23,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1733797608873.8650122bf6158d757da060b6cb41f4fe., hostname=d9f49988d155,36643,1733797608041, seqNum=2 from cache
2024-12-10T02:27:24,061 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:24,061 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:25,061 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:25,061 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:26,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:26,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:27,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:27,062 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:28,063 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:28,063 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:28,851 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-12-10T02:27:28,851 INFO  [master/d9f49988d155:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-12-10T02:27:29,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:29,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:30,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:30,064 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:31,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:31,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:32,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:32,065 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:33,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:33,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:33,807 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340
2024-12-10T02:27:34,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:34,066 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:35,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:35,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:36,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:36,067 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:37,068 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:37,068 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:38,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:38,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:39,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:39,069 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:40,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:40,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:41,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:41,070 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:42,071 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:42,071 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:43,071 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:43,071 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:43,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0095', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., hostname=d9f49988d155,36643,1733797608041, seqNum=122]
2024-12-10T02:27:44,072 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:44,072 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:45,073 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:45,073 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:45,492 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:27:45,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/75c5862e7f834709a8f54fac8ab32750 is 1080, key is row0095/info:/1733797663470/Put/seqid=0
2024-12-10T02:27:45,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741855_1031 (size=12513)
2024-12-10T02:27:45,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741855_1031 (size=12513)
2024-12-10T02:27:45,506 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/75c5862e7f834709a8f54fac8ab32750
2024-12-10T02:27:45,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/75c5862e7f834709a8f54fac8ab32750 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750
2024-12-10T02:27:45,518 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750, entries=7, sequenceid=132, filesize=12.2 K
2024-12-10T02:27:45,519 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 9703a4063d9cca5dbdf91fb53d597ddf in 27ms, sequenceid=132, compaction requested=false
2024-12-10T02:27:45,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:45,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:45,520 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-10T02:27:45,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/424040c1fb914dfebc1046c7fe58984f is 1080, key is row0102/info:/1733797665493/Put/seqid=0
2024-12-10T02:27:45,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741856_1032 (size=24394)
2024-12-10T02:27:45,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741856_1032 (size=24394)
2024-12-10T02:27:45,530 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/424040c1fb914dfebc1046c7fe58984f
2024-12-10T02:27:45,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/424040c1fb914dfebc1046c7fe58984f as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f
2024-12-10T02:27:45,538 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f, entries=18, sequenceid=153, filesize=23.8 K
2024-12-10T02:27:45,539 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=9.46 KB/9684 for 9703a4063d9cca5dbdf91fb53d597ddf in 19ms, sequenceid=153, compaction requested=true
2024-12-10T02:27:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:45,539 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:45,540 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 77737 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:45,540 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:45,541 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:45,541 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2f260b15a7504b4dac8ccc96b618baa8, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=75.9 K
2024-12-10T02:27:45,541 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f260b15a7504b4dac8ccc96b618baa8, keycount=33, bloomtype=ROW, size=39.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733797631215
2024-12-10T02:27:45,541 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c5862e7f834709a8f54fac8ab32750, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733797663470
2024-12-10T02:27:45,542 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424040c1fb914dfebc1046c7fe58984f, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733797665493
2024-12-10T02:27:45,553 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#71 average throughput is 19.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:45,553 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/ab36f3f6ea3d45c9860032ff0282f66b is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:45,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741857_1033 (size=67947)
2024-12-10T02:27:45,558 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741857_1033 (size=67947)
2024-12-10T02:27:45,563 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/ab36f3f6ea3d45c9860032ff0282f66b as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/ab36f3f6ea3d45c9860032ff0282f66b
2024-12-10T02:27:45,567 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into ab36f3f6ea3d45c9860032ff0282f66b(size=66.4 K), total size for store is 66.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:45,567 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:45,567 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797665539; duration=0sec
2024-12-10T02:27:45,567 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:45,568 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:46,073 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:46,073 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:47,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:47,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:47,537 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-12-10T02:27:47,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/24cee2e8ff7d475494d871ed1b1c4b94 is 1080, key is row0120/info:/1733797665521/Put/seqid=0
2024-12-10T02:27:47,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741858_1034 (size=15750)
2024-12-10T02:27:47,546 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741858_1034 (size=15750)
2024-12-10T02:27:47,547 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/24cee2e8ff7d475494d871ed1b1c4b94
2024-12-10T02:27:47,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/24cee2e8ff7d475494d871ed1b1c4b94 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94
2024-12-10T02:27:47,557 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94, entries=10, sequenceid=167, filesize=15.4 K
2024-12-10T02:27:47,558 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=14.71 KB/15064 for 9703a4063d9cca5dbdf91fb53d597ddf in 22ms, sequenceid=167, compaction requested=false
2024-12-10T02:27:47,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:47,560 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-12-10T02:27:47,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7145460541e14130995dc88508a76266 is 1080, key is row0130/info:/1733797667538/Put/seqid=0
2024-12-10T02:27:47,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741859_1035 (size=21156)
2024-12-10T02:27:47,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741859_1035 (size=21156)
2024-12-10T02:27:47,570 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7145460541e14130995dc88508a76266
2024-12-10T02:27:47,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7145460541e14130995dc88508a76266 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266
2024-12-10T02:27:47,580 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266, entries=15, sequenceid=185, filesize=20.7 K
2024-12-10T02:27:47,581 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=13.66 KB/13988 for 9703a4063d9cca5dbdf91fb53d597ddf in 21ms, sequenceid=185, compaction requested=true
2024-12-10T02:27:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:47,581 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:47,582 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T02:27:47,582 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 104853 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:47,582 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:47,582 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:47,583 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/ab36f3f6ea3d45c9860032ff0282f66b, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=102.4 K
2024-12-10T02:27:47,583 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab36f3f6ea3d45c9860032ff0282f66b, keycount=58, bloomtype=ROW, size=66.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733797631215
2024-12-10T02:27:47,583 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24cee2e8ff7d475494d871ed1b1c4b94, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733797665521
2024-12-10T02:27:47,584 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7145460541e14130995dc88508a76266, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733797667538
2024-12-10T02:27:47,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e21af13614a944a9aee834c1a033736c is 1080, key is row0145/info:/1733797667561/Put/seqid=0
2024-12-10T02:27:47,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741860_1036 (size=20078)
2024-12-10T02:27:47,591 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741860_1036 (size=20078)
2024-12-10T02:27:47,592 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e21af13614a944a9aee834c1a033736c
2024-12-10T02:27:47,597 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#75 average throughput is 42.59 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:47,597 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/cd2881e713604e789936830cc5b9b0fd is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:47,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e21af13614a944a9aee834c1a033736c as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c
2024-12-10T02:27:47,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741861_1037 (size=95076)
2024-12-10T02:27:47,602 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741861_1037 (size=95076)
2024-12-10T02:27:47,603 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c, entries=14, sequenceid=202, filesize=19.6 K
2024-12-10T02:27:47,604 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=2.10 KB/2152 for 9703a4063d9cca5dbdf91fb53d597ddf in 22ms, sequenceid=202, compaction requested=false
2024-12-10T02:27:47,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:47,607 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/cd2881e713604e789936830cc5b9b0fd as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/cd2881e713604e789936830cc5b9b0fd
2024-12-10T02:27:47,612 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into cd2881e713604e789936830cc5b9b0fd(size=92.8 K), total size for store is 112.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:47,612 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:47,613 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797667581; duration=0sec
2024-12-10T02:27:47,613 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:47,613 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:47,971 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-12-10T02:27:48,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:48,074 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:49,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:49,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:49,594 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:27:49,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c07b568e6d0a432c9f37f7134c1fd45a is 1080, key is row0159/info:/1733797667583/Put/seqid=0
2024-12-10T02:27:49,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741862_1038 (size=12516)
2024-12-10T02:27:49,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741862_1038 (size=12516)
2024-12-10T02:27:49,604 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c07b568e6d0a432c9f37f7134c1fd45a
2024-12-10T02:27:49,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c07b568e6d0a432c9f37f7134c1fd45a as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a
2024-12-10T02:27:49,614 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a, entries=7, sequenceid=213, filesize=12.2 K
2024-12-10T02:27:49,615 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 9703a4063d9cca5dbdf91fb53d597ddf in 20ms, sequenceid=213, compaction requested=true
2024-12-10T02:27:49,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:49,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:49,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:49,615 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:49,615 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-12-10T02:27:49,616 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 127670 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:49,616 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:49,616 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:49,616 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/cd2881e713604e789936830cc5b9b0fd, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=124.7 K
2024-12-10T02:27:49,617 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd2881e713604e789936830cc5b9b0fd, keycount=83, bloomtype=ROW, size=92.8 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733797631215
2024-12-10T02:27:49,617 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e21af13614a944a9aee834c1a033736c, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1733797667561
2024-12-10T02:27:49,618 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c07b568e6d0a432c9f37f7134c1fd45a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733797667583
2024-12-10T02:27:49,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/af82fe59197149e7b4363dc7521cb813 is 1080, key is row0166/info:/1733797669595/Put/seqid=0
2024-12-10T02:27:49,624 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741863_1039 (size=20078)
2024-12-10T02:27:49,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741863_1039 (size=20078)
2024-12-10T02:27:49,625 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/af82fe59197149e7b4363dc7521cb813
2024-12-10T02:27:49,630 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#78 average throughput is 53.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:49,631 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/51c609a60f67402780f2a24684a82d5f is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:49,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/af82fe59197149e7b4363dc7521cb813 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813
2024-12-10T02:27:49,637 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813, entries=14, sequenceid=230, filesize=19.6 K
2024-12-10T02:27:49,638 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 9703a4063d9cca5dbdf91fb53d597ddf in 23ms, sequenceid=230, compaction requested=false
2024-12-10T02:27:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:49,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741864_1040 (size=117820)
2024-12-10T02:27:49,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741864_1040 (size=117820)
2024-12-10T02:27:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:49,640 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-12-10T02:27:49,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e3569823585d44e4a07eece40c6089a8 is 1080, key is row0180/info:/1733797669616/Put/seqid=0
2024-12-10T02:27:49,644 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/51c609a60f67402780f2a24684a82d5f as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/51c609a60f67402780f2a24684a82d5f
2024-12-10T02:27:49,647 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741865_1041 (size=19000)
2024-12-10T02:27:49,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741865_1041 (size=19000)
2024-12-10T02:27:49,648 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e3569823585d44e4a07eece40c6089a8
2024-12-10T02:27:49,650 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into 51c609a60f67402780f2a24684a82d5f(size=115.1 K), total size for store is 134.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:49,650 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:49,650 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797669615; duration=0sec
2024-12-10T02:27:49,650 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:49,650 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:49,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/e3569823585d44e4a07eece40c6089a8 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8
2024-12-10T02:27:49,657 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8, entries=13, sequenceid=246, filesize=18.6 K
2024-12-10T02:27:49,658 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 9703a4063d9cca5dbdf91fb53d597ddf in 18ms, sequenceid=246, compaction requested=true
2024-12-10T02:27:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:49,658 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:49,659 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 156898 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:49,659 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:49,659 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:49,659 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/51c609a60f67402780f2a24684a82d5f, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=153.2 K
2024-12-10T02:27:49,659 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51c609a60f67402780f2a24684a82d5f, keycount=104, bloomtype=ROW, size=115.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733797631215
2024-12-10T02:27:49,660 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting af82fe59197149e7b4363dc7521cb813, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733797669595
2024-12-10T02:27:49,660 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3569823585d44e4a07eece40c6089a8, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733797669616
2024-12-10T02:27:49,669 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#80 average throughput is 67.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:49,669 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c0e6a7a8f742458ca48d137d33865fe5 is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:49,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741866_1042 (size=147233)
2024-12-10T02:27:49,673 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741866_1042 (size=147233)
2024-12-10T02:27:49,678 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c0e6a7a8f742458ca48d137d33865fe5 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c0e6a7a8f742458ca48d137d33865fe5
2024-12-10T02:27:49,683 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into c0e6a7a8f742458ca48d137d33865fe5(size=143.8 K), total size for store is 143.8 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:49,683 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:49,683 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797669658; duration=0sec
2024-12-10T02:27:49,683 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:49,683 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:50,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:50,075 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:51,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:51,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:51,651 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:27:51,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/07a93f9d9bf34bf39b948812c25b98a0 is 1080, key is row0193/info:/1733797671641/Put/seqid=0
2024-12-10T02:27:51,664 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741867_1043 (size=12518)
2024-12-10T02:27:51,665 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741867_1043 (size=12518)
2024-12-10T02:27:51,665 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/07a93f9d9bf34bf39b948812c25b98a0
2024-12-10T02:27:51,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/07a93f9d9bf34bf39b948812c25b98a0 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0
2024-12-10T02:27:51,676 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0, entries=7, sequenceid=258, filesize=12.2 K
2024-12-10T02:27:51,676 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=18.91 KB/19368 for 9703a4063d9cca5dbdf91fb53d597ddf in 26ms, sequenceid=258, compaction requested=false
2024-12-10T02:27:51,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:51,677 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=19.96 KB heapSize=21.63 KB
2024-12-10T02:27:51,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/a76b3e208f0e40519c6936356c0aaa1a is 1080, key is row0200/info:/1733797671651/Put/seqid=0
2024-12-10T02:27:51,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741868_1044 (size=25491)
2024-12-10T02:27:51,686 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741868_1044 (size=25491)
2024-12-10T02:27:51,686 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.96 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/a76b3e208f0e40519c6936356c0aaa1a
2024-12-10T02:27:51,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/a76b3e208f0e40519c6936356c0aaa1a as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a
2024-12-10T02:27:51,695 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a, entries=19, sequenceid=280, filesize=24.9 K
2024-12-10T02:27:51,695 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~19.96 KB/20444, heapSize ~21.61 KB/22128, currentSize=6.30 KB/6456 for 9703a4063d9cca5dbdf91fb53d597ddf in 18ms, sequenceid=280, compaction requested=true
2024-12-10T02:27:51,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:51,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:51,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:51,696 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:51,696 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 185242 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:51,697 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:51,697 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:51,697 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c0e6a7a8f742458ca48d137d33865fe5, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=180.9 K
2024-12-10T02:27:51,697 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0e6a7a8f742458ca48d137d33865fe5, keycount=131, bloomtype=ROW, size=143.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1733797631215
2024-12-10T02:27:51,697 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a93f9d9bf34bf39b948812c25b98a0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1733797671641
2024-12-10T02:27:51,698 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76b3e208f0e40519c6936356c0aaa1a, keycount=19, bloomtype=ROW, size=24.9 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733797671651
2024-12-10T02:27:51,708 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#83 average throughput is 80.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:51,709 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/2089bb4dcc2a4af4b59f460112c8be16 is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:51,711 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741869_1045 (size=175392)
2024-12-10T02:27:51,712 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741869_1045 (size=175392)
2024-12-10T02:27:51,716 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/2089bb4dcc2a4af4b59f460112c8be16 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2089bb4dcc2a4af4b59f460112c8be16
2024-12-10T02:27:51,721 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into 2089bb4dcc2a4af4b59f460112c8be16(size=171.3 K), total size for store is 171.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:51,722 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:51,722 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797671696; duration=0sec
2024-12-10T02:27:51,722 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:51,722 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:52,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:52,076 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:53,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:53,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:53,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:53,688 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-12-10T02:27:53,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/afb35377a1134de18bbfa2d99313b327 is 1080, key is row0219/info:/1733797671678/Put/seqid=0
2024-12-10T02:27:53,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741870_1046 (size=12523)
2024-12-10T02:27:53,702 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741870_1046 (size=12523)
2024-12-10T02:27:53,703 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/afb35377a1134de18bbfa2d99313b327
2024-12-10T02:27:53,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/afb35377a1134de18bbfa2d99313b327 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327
2024-12-10T02:27:53,713 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327, entries=7, sequenceid=291, filesize=12.2 K
2024-12-10T02:27:53,714 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 9703a4063d9cca5dbdf91fb53d597ddf in 26ms, sequenceid=291, compaction requested=false
2024-12-10T02:27:53,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:27:53,715 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB
2024-12-10T02:27:53,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7fbfde2611b74ccbaed65332006efb85 is 1080, key is row0226/info:/1733797673689/Put/seqid=0
2024-12-10T02:27:53,724 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741871_1047 (size=24412)
2024-12-10T02:27:53,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741871_1047 (size=24412)
2024-12-10T02:27:53,725 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7fbfde2611b74ccbaed65332006efb85
2024-12-10T02:27:53,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/7fbfde2611b74ccbaed65332006efb85 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85
2024-12-10T02:27:53,733 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-12-10T02:27:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:43062 deadline: 1733797683732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041
2024-12-10T02:27:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., hostname=d9f49988d155,36643,1733797608041, seqNum=122 , the old value is region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., hostname=d9f49988d155,36643,1733797608041, seqNum=122, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., hostname=d9f49988d155,36643,1733797608041, seqNum=122 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9703a4063d9cca5dbdf91fb53d597ddf, server=d9f49988d155,36643,1733797608041
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-12-10T02:27:53,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., hostname=d9f49988d155,36643,1733797608041, seqNum=122 because the exception is null or not the one we care about
2024-12-10T02:27:53,735 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85, entries=18, sequenceid=312, filesize=23.8 K
2024-12-10T02:27:53,736 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=11.56 KB/11836 for 9703a4063d9cca5dbdf91fb53d597ddf in 21ms, sequenceid=312, compaction requested=true
2024-12-10T02:27:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9703a4063d9cca5dbdf91fb53d597ddf:info, priority=-2147483648, current under compaction store size is 1
2024-12-10T02:27:53,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:53,736 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-12-10T02:27:53,737 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 212327 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-12-10T02:27:53,737 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1541): 9703a4063d9cca5dbdf91fb53d597ddf/info is initiating minor compaction (all files)
2024-12-10T02:27:53,737 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9703a4063d9cca5dbdf91fb53d597ddf/info in TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:27:53,737 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2089bb4dcc2a4af4b59f460112c8be16, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85] into tmpdir=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp, totalSize=207.4 K
2024-12-10T02:27:53,738 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2089bb4dcc2a4af4b59f460112c8be16, keycount=157, bloomtype=ROW, size=171.3 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733797631215
2024-12-10T02:27:53,738 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb35377a1134de18bbfa2d99313b327, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733797671678
2024-12-10T02:27:53,738 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fbfde2611b74ccbaed65332006efb85, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733797673689
2024-12-10T02:27:53,749 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9703a4063d9cca5dbdf91fb53d597ddf#info#compaction#86 average throughput is 93.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-12-10T02:27:53,749 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/9b729b83b3d6417caf8ecc21b589bf5a is 1080, key is row0062/info:/1733797631215/Put/seqid=0
2024-12-10T02:27:53,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741872_1048 (size=202473)
2024-12-10T02:27:53,752 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741872_1048 (size=202473)
2024-12-10T02:27:53,757 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/9b729b83b3d6417caf8ecc21b589bf5a as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/9b729b83b3d6417caf8ecc21b589bf5a
2024-12-10T02:27:53,762 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9703a4063d9cca5dbdf91fb53d597ddf/info of 9703a4063d9cca5dbdf91fb53d597ddf into 9b729b83b3d6417caf8ecc21b589bf5a(size=197.7 K), total size for store is 197.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-12-10T02:27:53,762 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:27:53,762 INFO  [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., storeName=9703a4063d9cca5dbdf91fb53d597ddf/info, priority=13, startTime=1733797673736; duration=0sec
2024-12-10T02:27:53,762 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-12-10T02:27:53,762 DEBUG [RS:0;d9f49988d155:36643-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9703a4063d9cca5dbdf91fb53d597ddf:info
2024-12-10T02:27:54,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:54,077 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:55,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:55,078 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:56,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:56,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:56,684 DEBUG [master/d9f49988d155:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=9, reused chunk count=70, reuseRatio=88.61%
2024-12-10T02:27:56,684 DEBUG [master/d9f49988d155:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0
2024-12-10T02:27:57,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:57,079 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:58,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:58,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:58,925 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9d68098034d65e6395ee8af43e3246e7, had cached 0 bytes from a total of 70862
2024-12-10T02:27:58,947 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 9703a4063d9cca5dbdf91fb53d597ddf, had cached 0 bytes from a total of 202473
2024-12-10T02:27:59,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:27:59,080 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:00,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:00,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:01,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:01,081 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:02,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:02,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:03,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:03,082 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,259 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,260 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,260 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,281 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,284 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,284 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,284 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,286 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36643 {}] regionserver.HRegion(8855): Flush requested on 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:28:03,776 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-12-10T02:28:03,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/33d8b8c6ca7449bcac34b83f8d440a05 is 1080, key is row0244/info:/1733797673716/Put/seqid=0
2024-12-10T02:28:03,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741873_1049 (size=17918)
2024-12-10T02:28:03,786 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741873_1049 (size=17918)
2024-12-10T02:28:03,788 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/33d8b8c6ca7449bcac34b83f8d440a05
2024-12-10T02:28:03,793 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-12-10T02:28:03,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/33d8b8c6ca7449bcac34b83f8d440a05 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/33d8b8c6ca7449bcac34b83f8d440a05
2024-12-10T02:28:03,794 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,794 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,795 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,796 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,796 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,798 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/33d8b8c6ca7449bcac34b83f8d440a05, entries=12, sequenceid=328, filesize=17.5 K
2024-12-10T02:28:03,799 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=1.05 KB/1076 for 9703a4063d9cca5dbdf91fb53d597ddf in 23ms, sequenceid=328, compaction requested=false
2024-12-10T02:28:03,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:28:03,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,815 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,816 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:03,821 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-12-10T02:28:04,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:04,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:05,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:05,083 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:05,778 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-12-10T02:28:05,779 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C36643%2C1733797608041.1733797685778
2024-12-10T02:28:05,795 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,795 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,795 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,795 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,795 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,795 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797608425 with entries=310, filesize=307.72 KB; new WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797685778
2024-12-10T02:28:05,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42801:42801),(127.0.0.1/127.0.0.1:35679:35679)]
2024-12-10T02:28:05,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797608425 is not closed yet, will try archiving it next time
2024-12-10T02:28:05,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741833_1009 (size=315114)
2024-12-10T02:28:05,797 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741833_1009 (size=315114)
2024-12-10T02:28:05,800 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 9703a4063d9cca5dbdf91fb53d597ddf 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-12-10T02:28:05,804 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c28f10993e13403d88c93654316fae1e is 1080, key is row0256/info:/1733797683777/Put/seqid=0
2024-12-10T02:28:05,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741875_1051 (size=6035)
2024-12-10T02:28:05,809 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741875_1051 (size=6035)
2024-12-10T02:28:05,809 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c28f10993e13403d88c93654316fae1e
2024-12-10T02:28:05,816 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/.tmp/info/c28f10993e13403d88c93654316fae1e as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c28f10993e13403d88c93654316fae1e
2024-12-10T02:28:05,822 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c28f10993e13403d88c93654316fae1e, entries=1, sequenceid=332, filesize=5.9 K
2024-12-10T02:28:05,823 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 9703a4063d9cca5dbdf91fb53d597ddf in 23ms, sequenceid=332, compaction requested=true
2024-12-10T02:28:05,823 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9703a4063d9cca5dbdf91fb53d597ddf:

2024-12-10T02:28:05,823 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB
2024-12-10T02:28:05,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/b1fe25daf43f4d65a5c446c32fc1f0ea is 193, key is TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf./info:regioninfo/1733797633978/Put/seqid=0
2024-12-10T02:28:05,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741876_1052 (size=6223)
2024-12-10T02:28:05,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741876_1052 (size=6223)
2024-12-10T02:28:05,833 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/b1fe25daf43f4d65a5c446c32fc1f0ea
2024-12-10T02:28:05,838 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/.tmp/info/b1fe25daf43f4d65a5c446c32fc1f0ea as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/info/b1fe25daf43f4d65a5c446c32fc1f0ea
2024-12-10T02:28:05,844 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/info/b1fe25daf43f4d65a5c446c32fc1f0ea, entries=5, sequenceid=21, filesize=6.1 K
2024-12-10T02:28:05,845 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 22ms, sequenceid=21, compaction requested=false
2024-12-10T02:28:05,845 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-12-10T02:28:05,845 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9d68098034d65e6395ee8af43e3246e7:

2024-12-10T02:28:05,845 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C36643%2C1733797608041.1733797685845
2024-12-10T02:28:05,852 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,852 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,853 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,853 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,853 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:05,853 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797685778 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797685845
2024-12-10T02:28:05,854 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741874_1050 (size=731)
2024-12-10T02:28:05,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741874_1050 (size=731)
2024-12-10T02:28:05,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35679:35679),(127.0.0.1/127.0.0.1:42801:42801)]
2024-12-10T02:28:05,857 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797608425 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs/d9f49988d155%2C36643%2C1733797608041.1733797608425
2024-12-10T02:28:05,857 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-12-10T02:28:05,857 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:28:05,858 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:28:05,858 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:05,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:05,858 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/WALs/d9f49988d155,36643,1733797608041/d9f49988d155%2C36643%2C1733797608041.1733797685778 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs/d9f49988d155%2C36643%2C1733797608041.1733797685778
2024-12-10T02:28:05,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:05,858 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:28:05,858 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:28:05,858 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1902379962, stopped=false
2024-12-10T02:28:05,858 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,46239,1733797607988
2024-12-10T02:28:05,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:05,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:05,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:05,860 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:28:05,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:05,861 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:05,861 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:28:05,861 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:05,861 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:05,861 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:05,862 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,36643,1733797608041' *****
2024-12-10T02:28:05,862 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:28:05,862 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:28:05,862 INFO  [RS:0;d9f49988d155:36643 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:28:05,862 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:28:05,862 INFO  [RS:0;d9f49988d155:36643 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:28:05,862 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(3091): Received CLOSE for 9703a4063d9cca5dbdf91fb53d597ddf
2024-12-10T02:28:05,862 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,36643,1733797608041
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:36643.
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9703a4063d9cca5dbdf91fb53d597ddf, disabling compactions & flushes
2024-12-10T02:28:05,863 DEBUG [RS:0;d9f49988d155:36643 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:05,863 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:28:05,863 DEBUG [RS:0;d9f49988d155:36643 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf. after waiting 0 ms
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:28:05,863 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-12-10T02:28:05,863 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1325): Online Regions={9703a4063d9cca5dbdf91fb53d597ddf=TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf., 1588230740=hbase:meta,,1.1588230740, 9d68098034d65e6395ee8af43e3246e7=TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.}
2024-12-10T02:28:05,863 DEBUG [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9703a4063d9cca5dbdf91fb53d597ddf, 9d68098034d65e6395ee8af43e3246e7
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:28:05,863 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:28:05,863 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:28:05,863 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-top, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2f260b15a7504b4dac8ccc96b618baa8, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/ab36f3f6ea3d45c9860032ff0282f66b, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/cd2881e713604e789936830cc5b9b0fd, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/51c609a60f67402780f2a24684a82d5f, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c0e6a7a8f742458ca48d137d33865fe5, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2089bb4dcc2a4af4b59f460112c8be16, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85] to archive
2024-12-10T02:28:05,865 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:28:05,870 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:28:05,873 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-d71d0757e17d4c8598102d5a7b9e93f0
2024-12-10T02:28:05,879 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2f260b15a7504b4dac8ccc96b618baa8 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2f260b15a7504b4dac8ccc96b618baa8
2024-12-10T02:28:05,880 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/TestLogRolling-testLogRolling=8650122bf6158d757da060b6cb41f4fe-648cb9d596cc447caf3ed23fb57c22e1
2024-12-10T02:28:05,881 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/75c5862e7f834709a8f54fac8ab32750
2024-12-10T02:28:05,882 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/424040c1fb914dfebc1046c7fe58984f
2024-12-10T02:28:05,882 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e21af13614a944a9aee834c1a033736c
2024-12-10T02:28:05,883 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/cd2881e713604e789936830cc5b9b0fd to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/cd2881e713604e789936830cc5b9b0fd
2024-12-10T02:28:05,883 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/24cee2e8ff7d475494d871ed1b1c4b94
2024-12-10T02:28:05,883 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7145460541e14130995dc88508a76266
2024-12-10T02:28:05,884 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/ab36f3f6ea3d45c9860032ff0282f66b to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/ab36f3f6ea3d45c9860032ff0282f66b
2024-12-10T02:28:05,885 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c07b568e6d0a432c9f37f7134c1fd45a
2024-12-10T02:28:05,885 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/51c609a60f67402780f2a24684a82d5f to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/51c609a60f67402780f2a24684a82d5f
2024-12-10T02:28:05,885 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-12-10T02:28:05,885 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c0e6a7a8f742458ca48d137d33865fe5 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/c0e6a7a8f742458ca48d137d33865fe5
2024-12-10T02:28:05,886 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:28:05,886 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:28:05,886 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797685863Running coprocessor pre-close hooks at 1733797685863Disabling compacts and flushes for region at 1733797685863Disabling writes for close at 1733797685863Writing region close event to WAL at 1733797685876 (+13 ms)Running coprocessor post-close hooks at 1733797685886 (+10 ms)Closed at 1733797685886
2024-12-10T02:28:05,886 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:28:05,886 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/af82fe59197149e7b4363dc7521cb813
2024-12-10T02:28:05,886 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/e3569823585d44e4a07eece40c6089a8
2024-12-10T02:28:05,886 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/a76b3e208f0e40519c6936356c0aaa1a
2024-12-10T02:28:05,887 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/07a93f9d9bf34bf39b948812c25b98a0
2024-12-10T02:28:05,887 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/afb35377a1134de18bbfa2d99313b327
2024-12-10T02:28:05,887 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2089bb4dcc2a4af4b59f460112c8be16 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/2089bb4dcc2a4af4b59f460112c8be16
2024-12-10T02:28:05,887 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85 to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/info/7fbfde2611b74ccbaed65332006efb85
2024-12-10T02:28:05,888 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d9f49988d155:46239 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-12-10T02:28:05,888 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [2f260b15a7504b4dac8ccc96b618baa8=40830, 75c5862e7f834709a8f54fac8ab32750=12513, ab36f3f6ea3d45c9860032ff0282f66b=67947, 424040c1fb914dfebc1046c7fe58984f=24394, 24cee2e8ff7d475494d871ed1b1c4b94=15750, cd2881e713604e789936830cc5b9b0fd=95076, 7145460541e14130995dc88508a76266=21156, e21af13614a944a9aee834c1a033736c=20078, 51c609a60f67402780f2a24684a82d5f=117820, c07b568e6d0a432c9f37f7134c1fd45a=12516, af82fe59197149e7b4363dc7521cb813=20078, c0e6a7a8f742458ca48d137d33865fe5=147233, e3569823585d44e4a07eece40c6089a8=19000, 07a93f9d9bf34bf39b948812c25b98a0=12518, 2089bb4dcc2a4af4b59f460112c8be16=175392, a76b3e208f0e40519c6936356c0aaa1a=25491, afb35377a1134de18bbfa2d99313b327=12523, 7fbfde2611b74ccbaed65332006efb85=24412]
2024-12-10T02:28:05,892 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9703a4063d9cca5dbdf91fb53d597ddf/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=121
2024-12-10T02:28:05,892 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:28:05,892 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9703a4063d9cca5dbdf91fb53d597ddf:
Waiting for close lock at 1733797685863Running coprocessor pre-close hooks at 1733797685863Disabling compacts and flushes for region at 1733797685863Disabling writes for close at 1733797685863Writing region close event to WAL at 1733797685889 (+26 ms)Running coprocessor post-close hooks at 1733797685892 (+3 ms)Closed at 1733797685892
2024-12-10T02:28:05,893 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1733797633292.9703a4063d9cca5dbdf91fb53d597ddf.
2024-12-10T02:28:05,893 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d68098034d65e6395ee8af43e3246e7, disabling compactions & flushes
2024-12-10T02:28:05,893 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:28:05,893 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:28:05,893 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7. after waiting 0 ms
2024-12-10T02:28:05,893 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:28:05,900 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe->hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/8650122bf6158d757da060b6cb41f4fe/info/90ae33ee017641dbacfddf352ac8bc56-bottom] to archive
2024-12-10T02:28:05,901 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.-1 {}] backup.HFileArchiver(363): Archiving compacted files.
2024-12-10T02:28:05,902 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe to hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/archive/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/info/90ae33ee017641dbacfddf352ac8bc56.8650122bf6158d757da060b6cb41f4fe
2024-12-10T02:28:05,903 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-12-10T02:28:05,906 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/data/default/TestLogRolling-testLogRolling/9d68098034d65e6395ee8af43e3246e7/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=121
2024-12-10T02:28:05,907 INFO  [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:28:05,907 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d68098034d65e6395ee8af43e3246e7:
Waiting for close lock at 1733797685893Running coprocessor pre-close hooks at 1733797685893Disabling compacts and flushes for region at 1733797685893Disabling writes for close at 1733797685893Writing region close event to WAL at 1733797685903 (+10 ms)Running coprocessor post-close hooks at 1733797685907 (+4 ms)Closed at 1733797685907
2024-12-10T02:28:05,907 DEBUG [RS_CLOSE_REGION-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1733797633292.9d68098034d65e6395ee8af43e3246e7.
2024-12-10T02:28:06,063 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,36643,1733797608041; all regions closed.
2024-12-10T02:28:06,064 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,064 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,064 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,064 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,064 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741834_1010 (size=8107)
2024-12-10T02:28:06,067 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741834_1010 (size=8107)
2024-12-10T02:28:06,069 DEBUG [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs
2024-12-10T02:28:06,069 INFO  [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C36643%2C1733797608041.meta:.meta(num 1733797608799)
2024-12-10T02:28:06,070 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,070 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,070 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,070 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,070 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741877_1053 (size=778)
2024-12-10T02:28:06,072 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741877_1053 (size=778)
2024-12-10T02:28:06,074 DEBUG [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/oldWALs
2024-12-10T02:28:06,074 INFO  [RS:0;d9f49988d155:36643 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C36643%2C1733797608041:(num 1733797685845)
2024-12-10T02:28:06,074 DEBUG [RS:0;d9f49988d155:36643 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:06,074 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:28:06,075 INFO  [RS:0;d9f49988d155:36643 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:28:06,075 INFO  [RS:0;d9f49988d155:36643 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-10T02:28:06,075 INFO  [RS:0;d9f49988d155:36643 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:28:06,075 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:28:06,075 INFO  [RS:0;d9f49988d155:36643 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36643
2024-12-10T02:28:06,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,36643,1733797608041
2024-12-10T02:28:06,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:28:06,077 INFO  [RS:0;d9f49988d155:36643 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:28:06,079 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,36643,1733797608041]
2024-12-10T02:28:06,084 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,36643,1733797608041 already deleted, retry=false
2024-12-10T02:28:06,084 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,36643,1733797608041 expired; onlineServers=0
2024-12-10T02:28:06,084 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,46239,1733797607988' *****
2024-12-10T02:28:06,084 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:28:06,084 INFO  [M:0;d9f49988d155:46239 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:28:06,084 INFO  [M:0;d9f49988d155:46239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:28:06,084 DEBUG [M:0;d9f49988d155:46239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:28:06,084 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:28:06,084 DEBUG [M:0;d9f49988d155:46239 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:28:06,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:06,084 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:06,084 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797608195 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797608195,5,FailOnTimeoutGroup]
2024-12-10T02:28:06,084 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797608195 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797608195,5,FailOnTimeoutGroup]
2024-12-10T02:28:06,085 INFO  [M:0;d9f49988d155:46239 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:28:06,085 INFO  [M:0;d9f49988d155:46239 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:28:06,085 DEBUG [M:0;d9f49988d155:46239 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:28:06,085 INFO  [M:0;d9f49988d155:46239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:28:06,085 INFO  [M:0;d9f49988d155:46239 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:28:06,085 INFO  [M:0;d9f49988d155:46239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:28:06,085 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:28:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:28:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:06,086 DEBUG [M:0;d9f49988d155:46239 {}] zookeeper.ZKUtil(347): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:28:06,086 WARN  [M:0;d9f49988d155:46239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:28:06,087 INFO  [M:0;d9f49988d155:46239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/.lastflushedseqids
2024-12-10T02:28:06,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741878_1054 (size=228)
2024-12-10T02:28:06,092 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741878_1054 (size=228)
2024-12-10T02:28:06,092 INFO  [M:0;d9f49988d155:46239 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:28:06,093 INFO  [M:0;d9f49988d155:46239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:28:06,093 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:28:06,093 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:06,093 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:06,093 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:28:06,093 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:06,093 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.42 KB heapSize=63.35 KB
2024-12-10T02:28:06,113 DEBUG [M:0;d9f49988d155:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d90a47190f634575a08aad373a594d09 is 82, key is hbase:meta,,1/info:regioninfo/1733797608823/Put/seqid=0
2024-12-10T02:28:06,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741879_1055 (size=5672)
2024-12-10T02:28:06,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741879_1055 (size=5672)
2024-12-10T02:28:06,119 INFO  [M:0;d9f49988d155:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d90a47190f634575a08aad373a594d09
2024-12-10T02:28:06,146 DEBUG [M:0;d9f49988d155:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d200945ad3fa4e33bd49f71965b373f0 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733797609236/Put/seqid=0
2024-12-10T02:28:06,151 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741880_1056 (size=7090)
2024-12-10T02:28:06,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741880_1056 (size=7090)
2024-12-10T02:28:06,152 INFO  [M:0;d9f49988d155:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.81 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d200945ad3fa4e33bd49f71965b373f0
2024-12-10T02:28:06,156 INFO  [M:0;d9f49988d155:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d200945ad3fa4e33bd49f71965b373f0
2024-12-10T02:28:06,172 DEBUG [M:0;d9f49988d155:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/becc08efbba044d6ae1e7f099c533914 is 69, key is d9f49988d155,36643,1733797608041/rs:state/1733797608280/Put/seqid=0
2024-12-10T02:28:06,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741881_1057 (size=5156)
2024-12-10T02:28:06,177 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741881_1057 (size=5156)
2024-12-10T02:28:06,177 INFO  [M:0;d9f49988d155:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/becc08efbba044d6ae1e7f099c533914
2024-12-10T02:28:06,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:06,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36643-0x1019a31e4b50001, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:06,179 INFO  [RS:0;d9f49988d155:36643 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:28:06,179 INFO  [RS:0;d9f49988d155:36643 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,36643,1733797608041; zookeeper connection closed.
2024-12-10T02:28:06,180 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@245dfab6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@245dfab6
2024-12-10T02:28:06,180 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:28:06,208 DEBUG [M:0;d9f49988d155:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9ee01843ef9545a5b3fe767f280b518f is 52, key is load_balancer_on/state:d/1733797608870/Put/seqid=0
2024-12-10T02:28:06,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741882_1058 (size=5056)
2024-12-10T02:28:06,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741882_1058 (size=5056)
2024-12-10T02:28:06,218 INFO  [M:0;d9f49988d155:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9ee01843ef9545a5b3fe767f280b518f
2024-12-10T02:28:06,224 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d90a47190f634575a08aad373a594d09 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d90a47190f634575a08aad373a594d09
2024-12-10T02:28:06,230 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d90a47190f634575a08aad373a594d09, entries=8, sequenceid=125, filesize=5.5 K
2024-12-10T02:28:06,231 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d200945ad3fa4e33bd49f71965b373f0 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d200945ad3fa4e33bd49f71965b373f0
2024-12-10T02:28:06,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:28:06,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-12-10T02:28:06,235 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-12-10T02:28:06,236 INFO  [M:0;d9f49988d155:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d200945ad3fa4e33bd49f71965b373f0
2024-12-10T02:28:06,236 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d200945ad3fa4e33bd49f71965b373f0, entries=13, sequenceid=125, filesize=6.9 K
2024-12-10T02:28:06,237 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/becc08efbba044d6ae1e7f099c533914 as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/becc08efbba044d6ae1e7f099c533914
2024-12-10T02:28:06,242 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/becc08efbba044d6ae1e7f099c533914, entries=1, sequenceid=125, filesize=5.0 K
2024-12-10T02:28:06,243 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/9ee01843ef9545a5b3fe767f280b518f as hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9ee01843ef9545a5b3fe767f280b518f
2024-12-10T02:28:06,248 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42629/user/jenkins/test-data/cc791a4e-c10d-817b-9437-16619496346e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/9ee01843ef9545a5b3fe767f280b518f, entries=1, sequenceid=125, filesize=4.9 K
2024-12-10T02:28:06,249 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=125, compaction requested=false
2024-12-10T02:28:06,257 INFO  [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:06,257 DEBUG [M:0;d9f49988d155:46239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797686093Disabling compacts and flushes for region at 1733797686093Disabling writes for close at 1733797686093Obtaining lock to block concurrent updates at 1733797686093Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797686093Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52651, getHeapSize=64808, getOffHeapSize=0, getCellsCount=148 at 1733797686094 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797686094Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797686094Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797686112 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797686112Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797686124 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797686146 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797686146Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797686157 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797686172 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797686172Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797686182 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797686207 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797686207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2623402c: reopening flushed file at 1733797686223 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9c19d1f: reopening flushed file at 1733797686230 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58232e65: reopening flushed file at 1733797686236 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354a7f1b: reopening flushed file at 1733797686242 (+6 ms)Finished flush of dataSize ~51.42 KB/52651, heapSize ~63.29 KB/64808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=125, compaction requested=false at 1733797686249 (+7 ms)Writing region close event to WAL at 1733797686257 (+8 ms)Closed at 1733797686257
2024-12-10T02:28:06,258 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,258 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,258 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,258 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,258 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:06,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43385 is added to blk_1073741830_1006 (size=61320)
2024-12-10T02:28:06,261 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32957 is added to blk_1073741830_1006 (size=61320)
2024-12-10T02:28:06,261 INFO  [M:0;d9f49988d155:46239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:28:06,262 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:28:06,262 INFO  [M:0;d9f49988d155:46239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46239
2024-12-10T02:28:06,262 INFO  [M:0;d9f49988d155:46239 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:28:06,295 INFO  [regionserver/d9f49988d155:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:28:06,364 INFO  [M:0;d9f49988d155:46239 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:28:06,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:06,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x1019a31e4b50000, quorum=127.0.0.1:52049, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:06,369 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5965a901{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:06,369 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6f87fe6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:06,369 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:06,370 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@191b8d86{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:06,370 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79ecb530{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:06,371 WARN  [BP-1725285643-172.17.0.2-1733797607278 heartbeating to localhost/127.0.0.1:42629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:28:06,371 WARN  [BP-1725285643-172.17.0.2-1733797607278 heartbeating to localhost/127.0.0.1:42629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725285643-172.17.0.2-1733797607278 (Datanode Uuid 7c1276fa-76aa-425d-807a-7d3259af4183) service to localhost/127.0.0.1:42629
2024-12-10T02:28:06,371 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:28:06,371 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:28:06,372 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data3/current/BP-1725285643-172.17.0.2-1733797607278 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:06,372 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data4/current/BP-1725285643-172.17.0.2-1733797607278 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:06,372 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:28:06,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@425d5d71{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:06,381 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4881a2ed{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:06,381 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:06,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@26d44036{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:06,381 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75b3fca0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:06,383 WARN  [BP-1725285643-172.17.0.2-1733797607278 heartbeating to localhost/127.0.0.1:42629 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:28:06,383 WARN  [BP-1725285643-172.17.0.2-1733797607278 heartbeating to localhost/127.0.0.1:42629 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1725285643-172.17.0.2-1733797607278 (Datanode Uuid 3e41693d-c85b-4b7a-86ed-cbf218d21a1f) service to localhost/127.0.0.1:42629
2024-12-10T02:28:06,384 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data1/current/BP-1725285643-172.17.0.2-1733797607278 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:06,384 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/cluster_bbb540c5-6cc4-086f-f0ff-5de8f46733e7/data/data2/current/BP-1725285643-172.17.0.2-1733797607278 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:06,384 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:28:06,384 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:28:06,385 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:28:06,393 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a48d3d4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:28:06,394 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0ef025{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:06,394 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:06,394 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@331b1dff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:06,394 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b7f954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:06,409 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:28:06,449 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:28:06,463 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=236 (was 210)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:42629
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-22
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-25
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42629
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42629 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:42629
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42629
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-18
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42629
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42629 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-24
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HFileArchiver-23
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:42629 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HFileArchiver-20
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42629
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=503 (was 485) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=20 (was 17) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4852 (was 3721) - AvailableMemoryMB LEAK? -
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=236, OpenFileDescriptor=503, MaxFileDescriptor=1048576, SystemLoadAverage=20, ProcessCount=11, AvailableMemoryMB=4850
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.log.dir so I do NOT create it in target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8b7e885e-f498-7c9d-4f31-a8f836a0befd/hadoop.tmp.dir so I do NOT create it in target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12, deleteOnExit=true
2024-12-10T02:28:06,473 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/test.cache.data in system properties and HBase conf
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.tmp.dir in system properties and HBase conf
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir in system properties and HBase conf
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/mapreduce.cluster.local.dir in system properties and HBase conf
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-12-10T02:28:06,474 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-12-10T02:28:06,474 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-12-10T02:28:06,475 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/nfs.dump.dir in system properties and HBase conf
2024-12-10T02:28:06,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/java.io.tmpdir in system properties and HBase conf
2024-12-10T02:28:06,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/dfs.journalnode.edits.dir in system properties and HBase conf
2024-12-10T02:28:06,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-12-10T02:28:06,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-12-10T02:28:06,496 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:28:06,575 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:28:06,579 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:28:06,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:28:06,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:28:06,597 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-12-10T02:28:06,598 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:28:06,599 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31586fd9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:28:06,599 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1853cb8d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:28:06,756 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ac034ca{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/java.io.tmpdir/jetty-localhost-33013-hadoop-hdfs-3_4_1-tests_jar-_-any-15413532774965200066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:28:06,756 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10c1adfc{HTTP/1.1, (http/1.1)}{localhost:33013}
2024-12-10T02:28:06,756 INFO  [Time-limited test {}] server.Server(415): Started @315815ms
2024-12-10T02:28:06,769 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-12-10T02:28:06,858 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:28:06,869 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:28:06,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:28:06,872 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:28:06,873 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:28:06,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@238bf9b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:28:06,873 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@814e400{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:28:06,988 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@372f7d77{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/java.io.tmpdir/jetty-localhost-45619-hadoop-hdfs-3_4_1-tests_jar-_-any-10615948613346975933/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:06,989 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3ffef2a8{HTTP/1.1, (http/1.1)}{localhost:45619}
2024-12-10T02:28:06,989 INFO  [Time-limited test {}] server.Server(415): Started @316047ms
2024-12-10T02:28:06,990 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:28:07,040 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-12-10T02:28:07,043 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-12-10T02:28:07,044 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-12-10T02:28:07,044 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-12-10T02:28:07,044 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-12-10T02:28:07,046 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@541a748b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,AVAILABLE}
2024-12-10T02:28:07,047 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4cb35637{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-12-10T02:28:07,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:07,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:07,129 WARN  [Thread-2477 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data2/current/BP-654835465-172.17.0.2-1733797686505/current, will proceed with Du for space computation calculation, 
2024-12-10T02:28:07,129 WARN  [Thread-2476 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data1/current/BP-654835465-172.17.0.2-1733797686505/current, will proceed with Du for space computation calculation, 
2024-12-10T02:28:07,158 WARN  [Thread-2455 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:28:07,160 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ff7d2c78c0030c8 with lease ID 0x330e34f507cc27d5: Processing first storage report for DS-b0b168dc-e035-42ef-a776-eae0057ddb1a from datanode DatanodeRegistration(127.0.0.1:39915, datanodeUuid=18357461-b821-4b01-a002-733b3349be32, infoPort=34415, infoSecurePort=0, ipcPort=39023, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505)
2024-12-10T02:28:07,160 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ff7d2c78c0030c8 with lease ID 0x330e34f507cc27d5: from storage DS-b0b168dc-e035-42ef-a776-eae0057ddb1a node DatanodeRegistration(127.0.0.1:39915, datanodeUuid=18357461-b821-4b01-a002-733b3349be32, infoPort=34415, infoSecurePort=0, ipcPort=39023, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:28:07,161 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ff7d2c78c0030c8 with lease ID 0x330e34f507cc27d5: Processing first storage report for DS-f946c9b5-934e-438d-a9fa-801bf52dd1d1 from datanode DatanodeRegistration(127.0.0.1:39915, datanodeUuid=18357461-b821-4b01-a002-733b3349be32, infoPort=34415, infoSecurePort=0, ipcPort=39023, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505)
2024-12-10T02:28:07,161 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ff7d2c78c0030c8 with lease ID 0x330e34f507cc27d5: from storage DS-f946c9b5-934e-438d-a9fa-801bf52dd1d1 node DatanodeRegistration(127.0.0.1:39915, datanodeUuid=18357461-b821-4b01-a002-733b3349be32, infoPort=34415, infoSecurePort=0, ipcPort=39023, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:28:07,202 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fea8446{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/java.io.tmpdir/jetty-localhost-38313-hadoop-hdfs-3_4_1-tests_jar-_-any-15038537081325863054/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:07,203 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f2be34d{HTTP/1.1, (http/1.1)}{localhost:38313}
2024-12-10T02:28:07,203 INFO  [Time-limited test {}] server.Server(415): Started @316261ms
2024-12-10T02:28:07,205 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-12-10T02:28:07,325 WARN  [Thread-2503 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data4/current/BP-654835465-172.17.0.2-1733797686505/current, will proceed with Du for space computation calculation, 
2024-12-10T02:28:07,332 WARN  [Thread-2502 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data3/current/BP-654835465-172.17.0.2-1733797686505/current, will proceed with Du for space computation calculation, 
2024-12-10T02:28:07,357 WARN  [Thread-2491 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-12-10T02:28:07,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcaa3fe2151359832 with lease ID 0x330e34f507cc27d6: Processing first storage report for DS-ae1e3be2-405f-48de-ba3d-670b5ed69054 from datanode DatanodeRegistration(127.0.0.1:39327, datanodeUuid=6ff595c8-6685-4f8b-bd95-16870dc1d5dc, infoPort=45567, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505)
2024-12-10T02:28:07,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcaa3fe2151359832 with lease ID 0x330e34f507cc27d6: from storage DS-ae1e3be2-405f-48de-ba3d-670b5ed69054 node DatanodeRegistration(127.0.0.1:39327, datanodeUuid=6ff595c8-6685-4f8b-bd95-16870dc1d5dc, infoPort=45567, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-12-10T02:28:07,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcaa3fe2151359832 with lease ID 0x330e34f507cc27d6: Processing first storage report for DS-71347229-423c-471b-9898-59a0ec37c845 from datanode DatanodeRegistration(127.0.0.1:39327, datanodeUuid=6ff595c8-6685-4f8b-bd95-16870dc1d5dc, infoPort=45567, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505)
2024-12-10T02:28:07,360 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcaa3fe2151359832 with lease ID 0x330e34f507cc27d6: from storage DS-71347229-423c-471b-9898-59a0ec37c845 node DatanodeRegistration(127.0.0.1:39327, datanodeUuid=6ff595c8-6685-4f8b-bd95-16870dc1d5dc, infoPort=45567, infoSecurePort=0, ipcPort=41927, storageInfo=lv=-57;cid=testClusterID;nsid=1435265522;c=1733797686505), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-12-10T02:28:07,439 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8
2024-12-10T02:28:07,442 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/zookeeper_0, clientPort=50780, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-10T02:28:07,443 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50780
2024-12-10T02:28:07,443 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,444 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:28:07,456 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741825_1001 (size=7)
2024-12-10T02:28:07,457 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f with version=8
2024-12-10T02:28:07,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:39613/user/jenkins/test-data/e22462c9-7599-3ef6-d6f3-c1528004a4c0/hbase-staging
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:28:07,460 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-10T02:28:07,461 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:28:07,461 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43455
2024-12-10T02:28:07,462 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43455 connecting to ZooKeeper ensemble=127.0.0.1:50780
2024-12-10T02:28:07,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434550x0, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:28:07,470 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43455-0x1019a331b210000 connected
2024-12-10T02:28:07,494 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,495 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:07,497 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f, hbase.cluster.distributed=false
2024-12-10T02:28:07,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:28:07,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43455
2024-12-10T02:28:07,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43455
2024-12-10T02:28:07,500 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43455
2024-12-10T02:28:07,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43455
2024-12-10T02:28:07,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43455
2024-12-10T02:28:07,520 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/d9f49988d155:0 server-side Connection retries=45
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-10T02:28:07,521 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-10T02:28:07,522 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38725
2024-12-10T02:28:07,523 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38725 connecting to ZooKeeper ensemble=127.0.0.1:50780
2024-12-10T02:28:07,523 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,525 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387250x0, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-10T02:28:07,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38725-0x1019a331b210001 connected
2024-12-10T02:28:07,531 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:07,531 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-10T02:28:07,531 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-10T02:28:07,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-10T02:28:07,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-10T02:28:07,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38725
2024-12-10T02:28:07,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38725
2024-12-10T02:28:07,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38725
2024-12-10T02:28:07,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38725
2024-12-10T02:28:07,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38725
2024-12-10T02:28:07,557 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d9f49988d155:43455
2024-12-10T02:28:07,557 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:28:07,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:28:07,560 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-10T02:28:07,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,562 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-10T02:28:07,562 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d9f49988d155,43455,1733797687460 from backup master directory
2024-12-10T02:28:07,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:28:07,565 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:28:07,565 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-10T02:28:07,573 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/hbase.id] with ID: 18fe6145-4e2a-4a01-9591-8f219d468fd9
2024-12-10T02:28:07,573 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/.tmp/hbase.id
2024-12-10T02:28:07,579 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:28:07,580 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741826_1002 (size=42)
2024-12-10T02:28:07,580 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/.tmp/hbase.id]:[hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/hbase.id]
2024-12-10T02:28:07,592 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:07,592 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-10T02:28:07,594 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-12-10T02:28:07,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:28:07,603 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741827_1003 (size=196)
2024-12-10T02:28:07,607 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-10T02:28:07,608 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-10T02:28:07,608 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:28:07,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:28:07,618 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741828_1004 (size=1189)
2024-12-10T02:28:07,619 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store
2024-12-10T02:28:07,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:28:07,631 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741829_1005 (size=34)
2024-12-10T02:28:07,632 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:28:07,633 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:28:07,633 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:07,633 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:07,633 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:28:07,633 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:07,633 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:07,633 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797687633Disabling compacts and flushes for region at 1733797687633Disabling writes for close at 1733797687633Writing region close event to WAL at 1733797687633Closed at 1733797687633
2024-12-10T02:28:07,634 WARN  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/.initializing
2024-12-10T02:28:07,634 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/WALs/d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,638 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C43455%2C1733797687460, suffix=, logDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/WALs/d9f49988d155,43455,1733797687460, archiveDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/oldWALs, maxLogs=10
2024-12-10T02:28:07,638 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C43455%2C1733797687460.1733797687638
2024-12-10T02:28:07,650 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/WALs/d9f49988d155,43455,1733797687460/d9f49988d155%2C43455%2C1733797687460.1733797687638
2024-12-10T02:28:07,656 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45567:45567),(127.0.0.1/127.0.0.1:34415:34415)]
2024-12-10T02:28:07,661 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:28:07,661 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:28:07,661 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,661 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,664 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:28:07,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-10T02:28:07,666 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:07,667 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:28:07,668 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-10T02:28:07,668 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,669 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:28:07,669 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:28:07,671 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-10T02:28:07,671 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,672 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:28:07,672 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-10T02:28:07,673 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-10T02:28:07,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,674 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-10T02:28:07,674 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,674 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,675 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,676 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,676 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,677 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-10T02:28:07,678 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-10T02:28:07,681 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:28:07,681 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752650, jitterRate=-0.0429568886756897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-10T02:28:07,682 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733797687661Initializing all the Stores at 1733797687662 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797687662Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797687664 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797687664Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797687664Cleaning up temporary data from old regions at 1733797687676 (+12 ms)Region opened successfully at 1733797687682 (+6 ms)
2024-12-10T02:28:07,684 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-10T02:28:07,687 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59152aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:28:07,688 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-10T02:28:07,689 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-10T02:28:07,689 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-10T02:28:07,689 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-10T02:28:07,689 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-10T02:28:07,690 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-10T02:28:07,690 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-10T02:28:07,698 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-10T02:28:07,699 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-10T02:28:07,701 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-10T02:28:07,701 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-10T02:28:07,702 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-10T02:28:07,703 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-10T02:28:07,703 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-10T02:28:07,705 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-10T02:28:07,708 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-10T02:28:07,709 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-10T02:28:07,710 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-10T02:28:07,713 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-10T02:28:07,714 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-10T02:28:07,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:07,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:07,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,717 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d9f49988d155,43455,1733797687460, sessionid=0x1019a331b210000, setting cluster-up flag (Was=false)
2024-12-10T02:28:07,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,728 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-10T02:28:07,730 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:07,741 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-10T02:28:07,742 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d9f49988d155,43455,1733797687460
2024-12-10T02:28:07,744 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-10T02:28:07,746 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-10T02:28:07,746 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-10T02:28:07,746 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-10T02:28:07,747 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d9f49988d155,43455,1733797687460
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d9f49988d155:0, corePoolSize=5, maxPoolSize=5
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d9f49988d155:0, corePoolSize=10, maxPoolSize=10
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:28:07,749 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:28:07,754 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-10T02:28:07,754 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733797717754
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-10T02:28:07,755 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-10T02:28:07,755 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,755 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-10T02:28:07,756 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,756 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-10T02:28:07,756 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-10T02:28:07,756 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-10T02:28:07,760 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-10T02:28:07,760 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-10T02:28:07,761 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797687760,5,FailOnTimeoutGroup]
2024-12-10T02:28:07,762 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797687761,5,FailOnTimeoutGroup]
2024-12-10T02:28:07,762 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,762 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-10T02:28:07,762 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,762 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,769 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:28:07,770 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741831_1007 (size=1321)
2024-12-10T02:28:07,770 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-10T02:28:07,770 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f
2024-12-10T02:28:07,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:28:07,782 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741832_1008 (size=32)
2024-12-10T02:28:07,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:28:07,784 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:28:07,785 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:28:07,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:07,786 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:28:07,787 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:28:07,787 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:07,788 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:28:07,789 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:28:07,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,790 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:07,790 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:28:07,791 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:28:07,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:07,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:07,791 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:28:07,792 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740
2024-12-10T02:28:07,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740
2024-12-10T02:28:07,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:28:07,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:28:07,794 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:28:07,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:28:07,798 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-10T02:28:07,798 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842765, jitterRate=0.07163174450397491}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:28:07,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733797687783Initializing all the Stores at 1733797687784 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797687784Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797687784Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797687784Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797687784Cleaning up temporary data from old regions at 1733797687794 (+10 ms)Region opened successfully at 1733797687798 (+4 ms)
2024-12-10T02:28:07,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:28:07,798 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:28:07,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:28:07,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:28:07,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:28:07,799 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:28:07,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797687798Disabling compacts and flushes for region at 1733797687798Disabling writes for close at 1733797687799 (+1 ms)Writing region close event to WAL at 1733797687799Closed at 1733797687799
2024-12-10T02:28:07,800 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:28:07,800 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-10T02:28:07,800 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-10T02:28:07,801 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:28:07,802 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-10T02:28:07,842 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(746): ClusterId : 18fe6145-4e2a-4a01-9591-8f219d468fd9
2024-12-10T02:28:07,842 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-10T02:28:07,845 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-10T02:28:07,845 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-10T02:28:07,847 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-10T02:28:07,847 DEBUG [RS:0;d9f49988d155:38725 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc4f96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d9f49988d155/172.17.0.2:0
2024-12-10T02:28:07,859 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d9f49988d155:38725
2024-12-10T02:28:07,859 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-10T02:28:07,859 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-10T02:28:07,860 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-10T02:28:07,860 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(2659): reportForDuty to master=d9f49988d155,43455,1733797687460 with port=38725, startcode=1733797687520
2024-12-10T02:28:07,861 DEBUG [RS:0;d9f49988d155:38725 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-10T02:28:07,864 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-12-10T02:28:07,865 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43455 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,865 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43455 {}] master.ServerManager(517): Registering regionserver=d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,867 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f
2024-12-10T02:28:07,867 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41535
2024-12-10T02:28:07,867 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-10T02:28:07,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:28:07,870 DEBUG [RS:0;d9f49988d155:38725 {}] zookeeper.ZKUtil(111): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,870 WARN  [RS:0;d9f49988d155:38725 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-10T02:28:07,870 INFO  [RS:0;d9f49988d155:38725 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:28:07,871 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,871 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d9f49988d155,38725,1733797687520]
2024-12-10T02:28:07,874 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-10T02:28:07,876 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-10T02:28:07,877 INFO  [RS:0;d9f49988d155:38725 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-10T02:28:07,877 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,877 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-10T02:28:07,878 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-10T02:28:07,878 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d9f49988d155:0, corePoolSize=2, maxPoolSize=2
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d9f49988d155:0, corePoolSize=1, maxPoolSize=1
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:28:07,878 DEBUG [RS:0;d9f49988d155:38725 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d9f49988d155:0, corePoolSize=3, maxPoolSize=3
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,879 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38725,1733797687520-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:28:07,895 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-10T02:28:07,895 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,38725,1733797687520-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,895 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,895 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.Replication(171): d9f49988d155,38725,1733797687520 started
2024-12-10T02:28:07,909 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:07,909 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1482): Serving as d9f49988d155,38725,1733797687520, RpcServer on d9f49988d155/172.17.0.2:38725, sessionid=0x1019a331b210001
2024-12-10T02:28:07,909 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-10T02:28:07,909 DEBUG [RS:0;d9f49988d155:38725 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,909 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,38725,1733797687520'
2024-12-10T02:28:07,909 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d9f49988d155,38725,1733797687520
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd9f49988d155,38725,1733797687520'
2024-12-10T02:28:07,910 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-10T02:28:07,911 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-10T02:28:07,911 DEBUG [RS:0;d9f49988d155:38725 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-10T02:28:07,911 INFO  [RS:0;d9f49988d155:38725 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-10T02:28:07,911 INFO  [RS:0;d9f49988d155:38725 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-10T02:28:07,953 WARN  [d9f49988d155:43455 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-10T02:28:08,013 INFO  [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C38725%2C1733797687520, suffix=, logDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/d9f49988d155,38725,1733797687520, archiveDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs, maxLogs=32
2024-12-10T02:28:08,014 INFO  [RS:0;d9f49988d155:38725 {}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C38725%2C1733797687520.1733797688013
2024-12-10T02:28:08,021 INFO  [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/d9f49988d155,38725,1733797687520/d9f49988d155%2C38725%2C1733797687520.1733797688013
2024-12-10T02:28:08,022 DEBUG [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45567:45567),(127.0.0.1/127.0.0.1:34415:34415)]
2024-12-10T02:28:08,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:08,085 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:08,203 DEBUG [d9f49988d155:43455 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-10T02:28:08,204 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d9f49988d155,38725,1733797687520
2024-12-10T02:28:08,205 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,38725,1733797687520, state=OPENING
2024-12-10T02:28:08,211 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-10T02:28:08,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:08,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:08,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:28:08,214 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-10T02:28:08,214 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,38725,1733797687520}]
2024-12-10T02:28:08,214 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:28:08,367 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-10T02:28:08,369 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43069, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-10T02:28:08,374 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-10T02:28:08,374 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:28:08,376 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d9f49988d155%2C38725%2C1733797687520.meta, suffix=.meta, logDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/d9f49988d155,38725,1733797687520, archiveDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs, maxLogs=32
2024-12-10T02:28:08,377 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor d9f49988d155%2C38725%2C1733797687520.meta.1733797688377.meta
2024-12-10T02:28:08,388 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/d9f49988d155,38725,1733797687520/d9f49988d155%2C38725%2C1733797687520.meta.1733797688377.meta
2024-12-10T02:28:08,401 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34415:34415),(127.0.0.1/127.0.0.1:45567:45567)]
2024-12-10T02:28:08,404 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-10T02:28:08,405 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-10T02:28:08,405 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-10T02:28:08,406 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-12-10T02:28:08,407 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-10T02:28:08,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:08,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:08,408 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-12-10T02:28:08,408 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-10T02:28:08,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:08,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:08,409 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-12-10T02:28:08,410 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-10T02:28:08,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:08,410 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:08,410 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-12-10T02:28:08,411 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-10T02:28:08,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-10T02:28:08,411 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-10T02:28:08,412 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-10T02:28:08,413 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740
2024-12-10T02:28:08,416 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740
2024-12-10T02:28:08,417 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-10T02:28:08,417 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-10T02:28:08,418 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-12-10T02:28:08,419 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-10T02:28:08,420 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=858145, jitterRate=0.09118854999542236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-12-10T02:28:08,420 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-10T02:28:08,421 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733797688405Writing region info on filesystem at 1733797688405Initializing all the Stores at 1733797688406 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797688406Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797688406Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733797688406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733797688406Cleaning up temporary data from old regions at 1733797688417 (+11 ms)Running coprocessor post-open hooks at 1733797688420 (+3 ms)Region opened successfully at 1733797688420
2024-12-10T02:28:08,422 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733797688366
2024-12-10T02:28:08,424 DEBUG [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-10T02:28:08,424 INFO  [RS_OPEN_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-10T02:28:08,425 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d9f49988d155,38725,1733797687520
2024-12-10T02:28:08,425 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d9f49988d155,38725,1733797687520, state=OPEN
2024-12-10T02:28:08,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:28:08,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-10T02:28:08,429 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d9f49988d155,38725,1733797687520
2024-12-10T02:28:08,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:28:08,430 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-10T02:28:08,432 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-10T02:28:08,432 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d9f49988d155,38725,1733797687520 in 215 msec
2024-12-10T02:28:08,434 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-10T02:28:08,434 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 632 msec
2024-12-10T02:28:08,435 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-10T02:28:08,435 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-10T02:28:08,437 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:28:08,437 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,38725,1733797687520, seqNum=-1]
2024-12-10T02:28:08,437 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:28:08,438 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46601, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:28:08,444 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 697 msec
2024-12-10T02:28:08,444 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733797688444, completionTime=-1
2024-12-10T02:28:08,445 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-12-10T02:28:08,445 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-10T02:28:08,447 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1
2024-12-10T02:28:08,447 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733797748447
2024-12-10T02:28:08,447 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733797808447
2024-12-10T02:28:08,447 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d9f49988d155:43455, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,448 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,450 DEBUG [master/d9f49988d155:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.888sec
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-10T02:28:08,453 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-10T02:28:08,455 DEBUG [master/d9f49988d155:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-10T02:28:08,455 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-10T02:28:08,455 INFO  [master/d9f49988d155:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d9f49988d155,43455,1733797687460-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-10T02:28:08,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c886945, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:28:08,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d9f49988d155,43455,-1 for getting cluster id
2024-12-10T02:28:08,543 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-10T02:28:08,544 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '18fe6145-4e2a-4a01-9591-8f219d468fd9'
2024-12-10T02:28:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-10T02:28:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "18fe6145-4e2a-4a01-9591-8f219d468fd9"

2024-12-10T02:28:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7cc10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:28:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d9f49988d155,43455,-1]
2024-12-10T02:28:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-10T02:28:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,547 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-10T02:28:08,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9e55fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-10T02:28:08,548 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-10T02:28:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d9f49988d155,38725,1733797687520, seqNum=-1]
2024-12-10T02:28:08,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-10T02:28:08,551 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-10T02:28:08,553 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d9f49988d155,43455,1733797687460
2024-12-10T02:28:08,553 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-12-10T02:28:08,556 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-12-10T02:28:08,557 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-12-10T02:28:08,559 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/test.com,8080,1, archiveDir=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs, maxLogs=32
2024-12-10T02:28:08,559 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733797688559
2024-12-10T02:28:08,569 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/test.com,8080,1/test.com%2C8080%2C1.1733797688559
2024-12-10T02:28:08,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45567:45567),(127.0.0.1/127.0.0.1:34415:34415)]
2024-12-10T02:28:08,576 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1733797688576
2024-12-10T02:28:08,590 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,590 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,590 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,590 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,590 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,590 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/test.com,8080,1/test.com%2C8080%2C1.1733797688559 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/test.com,8080,1/test.com%2C8080%2C1.1733797688576
2024-12-10T02:28:08,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741835_1011 (size=93)
2024-12-10T02:28:08,592 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741835_1011 (size=93)
2024-12-10T02:28:08,604 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/WALs/test.com,8080,1/test.com%2C8080%2C1.1733797688559 to hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs/test.com%2C8080%2C1.1733797688559
2024-12-10T02:28:08,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34415:34415),(127.0.0.1/127.0.0.1:45567:45567)]
2024-12-10T02:28:08,606 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,606 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,606 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,606 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,606 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,608 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741836_1012 (size=93)
2024-12-10T02:28:08,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741836_1012 (size=93)
2024-12-10T02:28:08,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs
2024-12-10T02:28:08,611 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1733797688576)
2024-12-10T02:28:08,611 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-10T02:28:08,612 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:28:08,612 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:08,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,612 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-10T02:28:08,612 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-10T02:28:08,612 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=79769275, stopped=false
2024-12-10T02:28:08,612 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d9f49988d155,43455,1733797687460
2024-12-10T02:28:08,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:08,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-10T02:28:08,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:08,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:08,616 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:28:08,616 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-10T02:28:08,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:08,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,617 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd9f49988d155,38725,1733797687520' *****
2024-12-10T02:28:08,617 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-10T02:28:08,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-10T02:28:08,617 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(959): stopping server d9f49988d155,38725,1733797687520
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:28:08,617 INFO  [RS:0;d9f49988d155:38725 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d9f49988d155:38725.
2024-12-10T02:28:08,618 DEBUG [RS:0;d9f49988d155:38725 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-10T02:28:08,618 DEBUG [RS:0;d9f49988d155:38725 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,618 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-10T02:28:08,618 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-10T02:28:08,618 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-10T02:28:08,618 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-10T02:28:08,618 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-10T02:28:08,618 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-10T02:28:08,618 DEBUG [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-10T02:28:08,618 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-10T02:28:08,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-10T02:28:08,619 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-10T02:28:08,619 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-10T02:28:08,619 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-10T02:28:08,619 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-10T02:28:08,619 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-12-10T02:28:08,643 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/.tmp/ns/e381f14995b44d068440909440e8d150 is 43, key is default/ns:d/1733797688439/Put/seqid=0
2024-12-10T02:28:08,648 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741837_1013 (size=5153)
2024-12-10T02:28:08,649 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741837_1013 (size=5153)
2024-12-10T02:28:08,650 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/.tmp/ns/e381f14995b44d068440909440e8d150
2024-12-10T02:28:08,655 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/.tmp/ns/e381f14995b44d068440909440e8d150 as hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/ns/e381f14995b44d068440909440e8d150
2024-12-10T02:28:08,660 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/ns/e381f14995b44d068440909440e8d150, entries=2, sequenceid=6, filesize=5.0 K
2024-12-10T02:28:08,661 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false
2024-12-10T02:28:08,661 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-10T02:28:08,664 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-10T02:28:08,665 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-10T02:28:08,665 INFO  [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-10T02:28:08,665 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733797688618Running coprocessor pre-close hooks at 1733797688618Disabling compacts and flushes for region at 1733797688618Disabling writes for close at 1733797688619 (+1 ms)Obtaining lock to block concurrent updates at 1733797688619Preparing flush snapshotting stores in 1588230740 at 1733797688619Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1733797688619Flushing stores of hbase:meta,,1.1588230740 at 1733797688620 (+1 ms)Flushing 1588230740/ns: creating writer at 1733797688620Flushing 1588230740/ns: appending metadata at 1733797688642 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733797688643 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a0ce81a: reopening flushed file at 1733797688655 (+12 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 42ms, sequenceid=6, compaction requested=false at 1733797688661 (+6 ms)Writing region close event to WAL at 1733797688662 (+1 ms)Running coprocessor post-close hooks at 1733797688665 (+3 ms)Closed at 1733797688665
2024-12-10T02:28:08,665 DEBUG [RS_CLOSE_META-regionserver/d9f49988d155:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-10T02:28:08,818 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(976): stopping server d9f49988d155,38725,1733797687520; all regions closed.
2024-12-10T02:28:08,819 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,819 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,819 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,819 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,819 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,823 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741834_1010 (size=1152)
2024-12-10T02:28:08,824 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741834_1010 (size=1152)
2024-12-10T02:28:08,828 DEBUG [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs
2024-12-10T02:28:08,828 INFO  [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C38725%2C1733797687520.meta:.meta(num 1733797688377)
2024-12-10T02:28:08,829 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,829 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,829 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,829 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,829 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741833_1009 (size=93)
2024-12-10T02:28:08,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741833_1009 (size=93)
2024-12-10T02:28:08,833 DEBUG [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/oldWALs
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog d9f49988d155%2C38725%2C1733797687520:(num 1733797688013)
2024-12-10T02:28:08,833 DEBUG [RS:0;d9f49988d155:38725 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.LeaseManager(133): Closed leases
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] hbase.ChoreService(370): Chore service for: regionserver/d9f49988d155:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:28:08,833 INFO  [regionserver/d9f49988d155:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:28:08,833 INFO  [RS:0;d9f49988d155:38725 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38725
2024-12-10T02:28:08,836 INFO  [RS:0;d9f49988d155:38725 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:28:08,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-10T02:28:08,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d9f49988d155,38725,1733797687520
2024-12-10T02:28:08,837 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d9f49988d155,38725,1733797687520]
2024-12-10T02:28:08,838 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d9f49988d155,38725,1733797687520 already deleted, retry=false
2024-12-10T02:28:08,838 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d9f49988d155,38725,1733797687520 expired; onlineServers=0
2024-12-10T02:28:08,838 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd9f49988d155,43455,1733797687460' *****
2024-12-10T02:28:08,838 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-10T02:28:08,839 DEBUG [M:0;d9f49988d155:43455 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-10T02:28:08,839 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-10T02:28:08,839 DEBUG [M:0;d9f49988d155:43455 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-10T02:28:08,839 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797687761 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.small.0-1733797687761,5,FailOnTimeoutGroup]
2024-12-10T02:28:08,839 DEBUG [master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797687760 {}] cleaner.HFileCleaner(306): Exit Thread[master/d9f49988d155:0:becomeActiveMaster-HFileCleaner.large.0-1733797687760,5,FailOnTimeoutGroup]
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] hbase.ChoreService(370): Chore service for: master/d9f49988d155:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-10T02:28:08,839 DEBUG [M:0;d9f49988d155:43455 {}] master.HMaster(1795): Stopping service threads
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-10T02:28:08,839 INFO  [M:0;d9f49988d155:43455 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-10T02:28:08,839 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-10T02:28:08,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-10T02:28:08,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-10T02:28:08,842 DEBUG [M:0;d9f49988d155:43455 {}] zookeeper.ZKUtil(347): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-10T02:28:08,842 WARN  [M:0;d9f49988d155:43455 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-10T02:28:08,843 INFO  [M:0;d9f49988d155:43455 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/.lastflushedseqids
2024-12-10T02:28:08,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741838_1014 (size=99)
2024-12-10T02:28:08,853 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741838_1014 (size=99)
2024-12-10T02:28:08,853 INFO  [M:0;d9f49988d155:43455 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-10T02:28:08,854 INFO  [M:0;d9f49988d155:43455 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-10T02:28:08,854 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-10T02:28:08,854 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:08,854 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:08,854 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-10T02:28:08,854 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:08,854 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-12-10T02:28:08,872 DEBUG [M:0;d9f49988d155:43455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9e5060b32f446259801cf3bade57c00 is 82, key is hbase:meta,,1/info:regioninfo/1733797688424/Put/seqid=0
2024-12-10T02:28:08,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741839_1015 (size=5672)
2024-12-10T02:28:08,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741839_1015 (size=5672)
2024-12-10T02:28:08,883 INFO  [M:0;d9f49988d155:43455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9e5060b32f446259801cf3bade57c00
2024-12-10T02:28:08,901 DEBUG [M:0;d9f49988d155:43455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0eedb26c56e451bb21f6cc179daf0e5 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1733797688443/Put/seqid=0
2024-12-10T02:28:08,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741840_1016 (size=5275)
2024-12-10T02:28:08,905 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741840_1016 (size=5275)
2024-12-10T02:28:08,906 INFO  [M:0;d9f49988d155:43455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0eedb26c56e451bb21f6cc179daf0e5
2024-12-10T02:28:08,933 DEBUG [M:0;d9f49988d155:43455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a901770089e248e2bf3dfcfd90f6d3a2 is 69, key is d9f49988d155,38725,1733797687520/rs:state/1733797687865/Put/seqid=0
2024-12-10T02:28:08,937 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741841_1017 (size=5156)
2024-12-10T02:28:08,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:08,937 INFO  [RS:0;d9f49988d155:38725 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:28:08,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38725-0x1019a331b210001, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:08,937 INFO  [RS:0;d9f49988d155:38725 {}] regionserver.HRegionServer(1031): Exiting; stopping=d9f49988d155,38725,1733797687520; zookeeper connection closed.
2024-12-10T02:28:08,938 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741841_1017 (size=5156)
2024-12-10T02:28:08,938 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3d860c59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3d860c59
2024-12-10T02:28:08,938 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-12-10T02:28:08,938 INFO  [M:0;d9f49988d155:43455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a901770089e248e2bf3dfcfd90f6d3a2
2024-12-10T02:28:08,958 DEBUG [M:0;d9f49988d155:43455 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f829ef5d10d44f1e8a523d943dc05f67 is 52, key is load_balancer_on/state:d/1733797688555/Put/seqid=0
2024-12-10T02:28:08,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741842_1018 (size=5056)
2024-12-10T02:28:08,964 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741842_1018 (size=5056)
2024-12-10T02:28:08,964 INFO  [M:0;d9f49988d155:43455 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f829ef5d10d44f1e8a523d943dc05f67
2024-12-10T02:28:08,969 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9e5060b32f446259801cf3bade57c00 as hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a9e5060b32f446259801cf3bade57c00
2024-12-10T02:28:08,973 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a9e5060b32f446259801cf3bade57c00, entries=8, sequenceid=29, filesize=5.5 K
2024-12-10T02:28:08,974 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f0eedb26c56e451bb21f6cc179daf0e5 as hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0eedb26c56e451bb21f6cc179daf0e5
2024-12-10T02:28:08,978 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f0eedb26c56e451bb21f6cc179daf0e5, entries=3, sequenceid=29, filesize=5.2 K
2024-12-10T02:28:08,979 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a901770089e248e2bf3dfcfd90f6d3a2 as hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a901770089e248e2bf3dfcfd90f6d3a2
2024-12-10T02:28:08,984 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a901770089e248e2bf3dfcfd90f6d3a2, entries=1, sequenceid=29, filesize=5.0 K
2024-12-10T02:28:08,984 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/f829ef5d10d44f1e8a523d943dc05f67 as hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f829ef5d10d44f1e8a523d943dc05f67
2024-12-10T02:28:08,988 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41535/user/jenkins/test-data/486e3ce2-9e7c-8696-2625-ef9a14f0a64f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/f829ef5d10d44f1e8a523d943dc05f67, entries=1, sequenceid=29, filesize=4.9 K
2024-12-10T02:28:08,990 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false
2024-12-10T02:28:08,992 INFO  [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-10T02:28:08,992 DEBUG [M:0;d9f49988d155:43455 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733797688854Disabling compacts and flushes for region at 1733797688854Disabling writes for close at 1733797688854Obtaining lock to block concurrent updates at 1733797688854Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733797688854Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1733797688855 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733797688855Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733797688855Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733797688871 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733797688871Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733797688886 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733797688900 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733797688900Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733797688910 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733797688932 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733797688932Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733797688943 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733797688957 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733797688957Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d8642f3: reopening flushed file at 1733797688968 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d9d528c: reopening flushed file at 1733797688973 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6704eba: reopening flushed file at 1733797688978 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b4230ba: reopening flushed file at 1733797688984 (+6 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 135ms, sequenceid=29, compaction requested=false at 1733797688990 (+6 ms)Writing region close event to WAL at 1733797688992 (+2 ms)Closed at 1733797688992
2024-12-10T02:28:08,993 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,993 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,993 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,993 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,994 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-12-10T02:28:08,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39915 is added to blk_1073741830_1006 (size=10311)
2024-12-10T02:28:08,995 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39327 is added to blk_1073741830_1006 (size=10311)
2024-12-10T02:28:08,997 INFO  [M:0;d9f49988d155:43455 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-10T02:28:08,997 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-10T02:28:08,997 INFO  [M:0;d9f49988d155:43455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43455
2024-12-10T02:28:08,997 INFO  [M:0;d9f49988d155:43455 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-10T02:28:09,086 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,40537,1733797475089/d9f49988d155%2C40537%2C1733797475089.meta.1733797475892.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:09,086 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:42523/user/jenkins/test-data/4f54d8f1-2a09-2cec-00f9-22253a10399e/WALs/d9f49988d155,33309,1733797476037/d9f49988d155%2C33309%2C1733797476037.1733797476224
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-12-10T02:28:09,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:09,099 INFO  [M:0;d9f49988d155:43455 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-10T02:28:09,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43455-0x1019a331b210000, quorum=127.0.0.1:50780, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-10T02:28:09,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fea8446{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:09,102 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f2be34d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:09,102 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:09,102 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4cb35637{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:09,102 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@541a748b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:09,103 WARN  [BP-654835465-172.17.0.2-1733797686505 heartbeating to localhost/127.0.0.1:41535 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:28:09,103 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:28:09,103 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:28:09,103 WARN  [BP-654835465-172.17.0.2-1733797686505 heartbeating to localhost/127.0.0.1:41535 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-654835465-172.17.0.2-1733797686505 (Datanode Uuid 6ff595c8-6685-4f8b-bd95-16870dc1d5dc) service to localhost/127.0.0.1:41535
2024-12-10T02:28:09,104 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data3/current/BP-654835465-172.17.0.2-1733797686505 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:09,104 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data4/current/BP-654835465-172.17.0.2-1733797686505 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:09,104 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:28:09,110 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@372f7d77{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-12-10T02:28:09,111 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3ffef2a8{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:09,111 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:09,111 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@814e400{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:09,111 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@238bf9b3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:09,114 WARN  [BP-654835465-172.17.0.2-1733797686505 heartbeating to localhost/127.0.0.1:41535 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-12-10T02:28:09,114 WARN  [BP-654835465-172.17.0.2-1733797686505 heartbeating to localhost/127.0.0.1:41535 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-654835465-172.17.0.2-1733797686505 (Datanode Uuid 18357461-b821-4b01-a002-733b3349be32) service to localhost/127.0.0.1:41535
2024-12-10T02:28:09,114 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-12-10T02:28:09,114 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-12-10T02:28:09,114 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data1/current/BP-654835465-172.17.0.2-1733797686505 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:09,115 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/cluster_5d62ae0c-41a0-c3f4-c5cb-dd82ee481b12/data/data2/current/BP-654835465-172.17.0.2-1733797686505 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-12-10T02:28:09,115 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-12-10T02:28:09,120 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ac034ca{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-12-10T02:28:09,121 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10c1adfc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-12-10T02:28:09,121 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-12-10T02:28:09,121 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1853cb8d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-12-10T02:28:09,121 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31586fd9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dfdbab02-1bfa-7139-a524-d2423b367fd8/hadoop.log.dir/,STOPPED}
2024-12-10T02:28:09,127 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-10T02:28:09,144 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-10T02:28:09,155 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=277 (was 236)
Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:41535
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-21
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:41535
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41535 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41535 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41535
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41535 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41535
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41535
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 503) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=20 (was 20), ProcessCount=11 (was 11), AvailableMemoryMB=4608 (was 4850)