2024-11-16 16:33:44,952 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-16 16:33:44,968 main DEBUG Took 0.013132 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-16 16:33:44,968 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-16 16:33:44,969 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-16 16:33:44,970 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-16 16:33:44,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:44,979 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-16 16:33:44,996 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:44,998 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:44,998 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:44,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,000 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,004 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,006 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,008 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,009 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,010 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,011 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,011 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-16 16:33:45,011 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,012 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-16 16:33:45,014 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-16 16:33:45,015 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-16 16:33:45,017 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-16 16:33:45,018 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-16 16:33:45,020 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-16 16:33:45,020 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-16 16:33:45,030 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-16 16:33:45,034 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-16 16:33:45,036 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-16 16:33:45,036 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-16 16:33:45,037 main DEBUG createAppenders(={Console})
2024-11-16 16:33:45,038 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-16 16:33:45,038 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-16 16:33:45,038 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-16 16:33:45,039 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-16 16:33:45,039 main DEBUG OutputStream closed
2024-11-16 16:33:45,039 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-16 16:33:45,040 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-16 16:33:45,040 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-16 16:33:45,128 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-16 16:33:45,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-16 16:33:45,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-16 16:33:45,134 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-16 16:33:45,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-16 16:33:45,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-16 16:33:45,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-16 16:33:45,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-16 16:33:45,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-16 16:33:45,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-16 16:33:45,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-16 16:33:45,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-16 16:33:45,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-16 16:33:45,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-16 16:33:45,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-16 16:33:45,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-16 16:33:45,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-16 16:33:45,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-16 16:33:45,143 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-16 16:33:45,144 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-16 16:33:45,144 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-16 16:33:45,145 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-16T16:33:45,381 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c
2024-11-16 16:33:45,385 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-16 16:33:45,385 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-16T16:33:45,398 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-16T16:33:45,437 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=5081
2024-11-16T16:33:45,440 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:33:45,456 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8, deleteOnExit=true
2024-11-16T16:33:45,456 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:33:45,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/test.cache.data in system properties and HBase conf
2024-11-16T16:33:45,457 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:33:45,458 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:33:45,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:33:45,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:33:45,459 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:33:45,561 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-16T16:33:45,676 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:33:45,682 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:33:45,683 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:33:45,684 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:33:45,685 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:33:45,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:33:45,686 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:33:45,687 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:33:45,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:33:45,688 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:33:45,689 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:33:45,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:33:45,690 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:33:45,691 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:33:45,692 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:33:46,174 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:33:46,971 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-16T16:33:47,038 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2813ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-16T16:33:47,109 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:33:47,171 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:33:47,189 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:33:47,190 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:33:47,191 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:33:47,203 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:33:47,206 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:33:47,207 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:33:47,428 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6de997b9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/java.io.tmpdir/jetty-localhost-35735-hadoop-hdfs-3_4_1-tests_jar-_-any-5931418293530828198/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:33:47,434 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:35735}
2024-11-16T16:33:47,434 INFO  [Time-limited test {}] server.Server(415): Started @3210ms
2024-11-16T16:33:47,463 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:33:47,981 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:33:47,989 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:33:47,990 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:33:47,990 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:33:47,991 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:33:47,992 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:33:47,992 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:33:48,094 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f93babe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/java.io.tmpdir/jetty-localhost-40619-hadoop-hdfs-3_4_1-tests_jar-_-any-6815154429867968458/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:33:48,094 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:40619}
2024-11-16T16:33:48,095 INFO  [Time-limited test {}] server.Server(415): Started @3870ms
2024-11-16T16:33:48,148 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:33:48,258 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:33:48,266 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:33:48,268 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:33:48,268 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:33:48,269 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:33:48,270 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:33:48,271 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:33:48,407 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6c963ecd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/java.io.tmpdir/jetty-localhost-36761-hadoop-hdfs-3_4_1-tests_jar-_-any-14910333408550648677/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:33:48,408 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:36761}
2024-11-16T16:33:48,408 INFO  [Time-limited test {}] server.Server(415): Started @4184ms
2024-11-16T16:33:48,410 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:33:49,835 WARN  [Thread-101 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data4/current/BP-908077221-172.17.0.2-1731774826254/current, will proceed with Du for space computation calculation, 
2024-11-16T16:33:49,835 WARN  [Thread-99 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data2/current/BP-908077221-172.17.0.2-1731774826254/current, will proceed with Du for space computation calculation, 
2024-11-16T16:33:49,835 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data1/current/BP-908077221-172.17.0.2-1731774826254/current, will proceed with Du for space computation calculation, 
2024-11-16T16:33:49,835 WARN  [Thread-100 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data3/current/BP-908077221-172.17.0.2-1731774826254/current, will proceed with Du for space computation calculation, 
2024-11-16T16:33:49,868 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:33:49,868 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:33:49,914 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe14fb876d82a6040 with lease ID 0x9d42e9df3778f84: Processing first storage report for DS-db944295-0531-4e0c-baa5-a6c1f976c343 from datanode DatanodeRegistration(127.0.0.1:39383, datanodeUuid=c27172ad-74de-4f82-ab3b-bf7b53f0c1f6, infoPort=37775, infoSecurePort=0, ipcPort=42095, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254)
2024-11-16T16:33:49,915 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe14fb876d82a6040 with lease ID 0x9d42e9df3778f84: from storage DS-db944295-0531-4e0c-baa5-a6c1f976c343 node DatanodeRegistration(127.0.0.1:39383, datanodeUuid=c27172ad-74de-4f82-ab3b-bf7b53f0c1f6, infoPort=37775, infoSecurePort=0, ipcPort=42095, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-16T16:33:49,916 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64141e75be88347 with lease ID 0x9d42e9df3778f83: Processing first storage report for DS-a01e59ff-95a4-4449-a034-2687d1d4f202 from datanode DatanodeRegistration(127.0.0.1:45495, datanodeUuid=01a12fe5-d614-411c-8468-38ddaa27a747, infoPort=44591, infoSecurePort=0, ipcPort=34857, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254)
2024-11-16T16:33:49,916 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64141e75be88347 with lease ID 0x9d42e9df3778f83: from storage DS-a01e59ff-95a4-4449-a034-2687d1d4f202 node DatanodeRegistration(127.0.0.1:45495, datanodeUuid=01a12fe5-d614-411c-8468-38ddaa27a747, infoPort=44591, infoSecurePort=0, ipcPort=34857, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:33:49,916 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe14fb876d82a6040 with lease ID 0x9d42e9df3778f84: Processing first storage report for DS-3f18b40d-72b0-4a89-94de-ae2c9f9b9008 from datanode DatanodeRegistration(127.0.0.1:39383, datanodeUuid=c27172ad-74de-4f82-ab3b-bf7b53f0c1f6, infoPort=37775, infoSecurePort=0, ipcPort=42095, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254)
2024-11-16T16:33:49,917 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe14fb876d82a6040 with lease ID 0x9d42e9df3778f84: from storage DS-3f18b40d-72b0-4a89-94de-ae2c9f9b9008 node DatanodeRegistration(127.0.0.1:39383, datanodeUuid=c27172ad-74de-4f82-ab3b-bf7b53f0c1f6, infoPort=37775, infoSecurePort=0, ipcPort=42095, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:33:49,917 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64141e75be88347 with lease ID 0x9d42e9df3778f83: Processing first storage report for DS-f582371e-a30a-4c88-a655-c7fa5714bd93 from datanode DatanodeRegistration(127.0.0.1:45495, datanodeUuid=01a12fe5-d614-411c-8468-38ddaa27a747, infoPort=44591, infoSecurePort=0, ipcPort=34857, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254)
2024-11-16T16:33:49,917 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64141e75be88347 with lease ID 0x9d42e9df3778f83: from storage DS-f582371e-a30a-4c88-a655-c7fa5714bd93 node DatanodeRegistration(127.0.0.1:45495, datanodeUuid=01a12fe5-d614-411c-8468-38ddaa27a747, infoPort=44591, infoSecurePort=0, ipcPort=34857, storageInfo=lv=-57;cid=testClusterID;nsid=379369049;c=1731774826254), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:33:49,924 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c
2024-11-16T16:33:49,997 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/zookeeper_0, clientPort=51607, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:33:50,007 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51607
2024-11-16T16:33:50,021 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:50,024 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:50,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:33:50,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:33:50,656 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c with version=8
2024-11-16T16:33:50,656 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:33:50,729 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-16T16:33:50,991 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:33:51,001 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,001 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,005 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:33:51,005 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,006 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:33:51,135 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:33:51,192 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-16T16:33:51,201 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-16T16:33:51,205 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:33:51,228 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 103274 (auto-detected)
2024-11-16T16:33:51,229 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-11-16T16:33:51,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39093
2024-11-16T16:33:51,267 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39093 connecting to ZooKeeper ensemble=127.0.0.1:51607
2024-11-16T16:33:51,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390930x0, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:33:51,407 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39093-0x101447ed1480000 connected
2024-11-16T16:33:51,504 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:51,507 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:51,517 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:33:51,520 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c, hbase.cluster.distributed=false
2024-11-16T16:33:51,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:33:51,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39093
2024-11-16T16:33:51,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39093
2024-11-16T16:33:51,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39093
2024-11-16T16:33:51,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39093
2024-11-16T16:33:51,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39093
2024-11-16T16:33:51,646 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:33:51,647 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,647 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,648 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:33:51,648 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:33:51,648 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:33:51,650 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:33:51,653 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:33:51,654 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35937
2024-11-16T16:33:51,656 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35937 connecting to ZooKeeper ensemble=127.0.0.1:51607
2024-11-16T16:33:51,657 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:51,661 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:51,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359370x0, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:33:51,680 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359370x0, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:33:51,680 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35937-0x101447ed1480001 connected
2024-11-16T16:33:51,684 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:33:51,690 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:33:51,693 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:33:51,697 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:33:51,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35937
2024-11-16T16:33:51,698 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35937
2024-11-16T16:33:51,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35937
2024-11-16T16:33:51,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35937
2024-11-16T16:33:51,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35937
2024-11-16T16:33:51,714 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:39093
2024-11-16T16:33:51,715 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,39093,1731774830835
2024-11-16T16:33:51,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:33:51,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:33:51,734 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,39093,1731774830835
2024-11-16T16:33:51,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:33:51,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:51,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:51,764 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:33:51,765 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,39093,1731774830835 from backup master directory
2024-11-16T16:33:51,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,39093,1731774830835
2024-11-16T16:33:51,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:33:51,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:33:51,774 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:33:51,774 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,39093,1731774830835
2024-11-16T16:33:51,776 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-16T16:33:51,777 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-16T16:33:51,831 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase.id] with ID: 0f53d255-655e-45fb-b61e-3fec7d6cedbd
2024-11-16T16:33:51,831 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/.tmp/hbase.id
2024-11-16T16:33:51,843 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:33:51,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:33:51,846 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/.tmp/hbase.id]:[hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase.id]
2024-11-16T16:33:51,887 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:51,893 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:33:51,911 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms.
2024-11-16T16:33:51,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:51,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:51,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:33:51,959 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:33:51,972 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:33:51,973 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:33:51,978 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:33:52,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:33:52,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:33:52,022 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store
2024-11-16T16:33:52,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:33:52,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:33:52,046 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-16T16:33:52,050 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:52,051 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:33:52,051 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:33:52,051 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:33:52,053 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:33:52,053 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:33:52,053 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:33:52,054 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774832051Disabling compacts and flushes for region at 1731774832051Disabling writes for close at 1731774832053 (+2 ms)Writing region close event to WAL at 1731774832053Closed at 1731774832053
2024-11-16T16:33:52,056 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/.initializing
2024-11-16T16:33:52,057 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/WALs/303fe01965e8,39093,1731774830835
2024-11-16T16:33:52,076 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C39093%2C1731774830835, suffix=, logDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/WALs/303fe01965e8,39093,1731774830835, archiveDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/oldWALs, maxLogs=10
2024-11-16T16:33:52,084 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C39093%2C1731774830835.1731774832080
2024-11-16T16:33:52,101 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/WALs/303fe01965e8,39093,1731774830835/303fe01965e8%2C39093%2C1731774830835.1731774832080
2024-11-16T16:33:52,109 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44591:44591),(127.0.0.1/127.0.0.1:37775:37775)]
2024-11-16T16:33:52,111 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:33:52,111 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:52,114 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,115 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,147 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:33:52,169 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:33:52,172 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,175 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:52,175 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:33:52,178 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:33:52,178 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:33:52,179 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:33:52,182 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:33:52,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,184 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:33:52,184 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:33:52,186 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:33:52,187 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,187 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:33:52,188 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,192 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,193 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,199 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,200 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,204 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:33:52,210 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:33:52,215 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:33:52,216 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=762539, jitterRate=-0.030382394790649414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:33:52,226 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731774832126Initializing all the Stores at 1731774832128 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774832128Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774832129 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774832129Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774832129Cleaning up temporary data from old regions at 1731774832200 (+71 ms)Region opened successfully at 1731774832226 (+26 ms)
2024-11-16T16:33:52,228 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:33:52,267 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb5d7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:33:52,296 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:33:52,306 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:33:52,307 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:33:52,309 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:33:52,310 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-16T16:33:52,315 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-16T16:33:52,315 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:33:52,338 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:33:52,346 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:33:52,417 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:33:52,420 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:33:52,423 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:33:52,447 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:33:52,451 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:33:52,457 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:33:52,468 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:33:52,470 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:33:52,479 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:33:52,497 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:33:52,508 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:33:52,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:33:52,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:33:52,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,526 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,39093,1731774830835, sessionid=0x101447ed1480000, setting cluster-up flag (Was=false)
2024-11-16T16:33:52,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,584 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:33:52,587 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39093,1731774830835
2024-11-16T16:33:52,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:52,637 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:33:52,640 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39093,1731774830835
2024-11-16T16:33:52,649 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:33:52,705 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(746): ClusterId : 0f53d255-655e-45fb-b61e-3fec7d6cedbd
2024-11-16T16:33:52,708 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:33:52,714 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:33:52,723 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:33:52,723 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:33:52,723 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:33:52,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:33:52,733 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:33:52,734 DEBUG [RS:0;303fe01965e8:35937 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a93e37d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:33:52,733 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,39093,1731774830835
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:33:52,740 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:33:52,741 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:33:52,741 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:33:52,741 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:33:52,741 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:33:52,741 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,742 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:33:52,742 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,743 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731774862743
2024-11-16T16:33:52,745 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:33:52,746 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:33:52,746 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:33:52,747 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:33:52,750 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:35937
2024-11-16T16:33:52,750 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:33:52,750 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:33:52,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:33:52,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:33:52,752 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,752 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:33:52,752 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,754 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:33:52,754 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:33:52,755 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:33:52,755 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:33:52,756 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:33:52,757 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:33:52,758 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,39093,1731774830835 with port=35937, startcode=1731774831611
2024-11-16T16:33:52,759 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:33:52,759 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:33:52,761 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774832760,5,FailOnTimeoutGroup]
2024-11-16T16:33:52,761 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774832761,5,FailOnTimeoutGroup]
2024-11-16T16:33:52,762 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,762 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:33:52,763 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,763 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:33:52,764 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,764 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:33:52,766 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:33:52,766 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c
2024-11-16T16:33:52,773 DEBUG [RS:0;303fe01965e8:35937 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:33:52,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:33:52,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:33:52,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:52,784 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:33:52,786 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:33:52,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,787 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:52,787 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:33:52,790 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:33:52,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:52,791 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:33:52,794 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:33:52,794 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,795 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:52,796 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:33:52,799 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:33:52,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:52,801 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:52,801 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:33:52,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740
2024-11-16T16:33:52,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740
2024-11-16T16:33:52,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:33:52,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:33:52,808 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:33:52,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:33:52,816 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:33:52,817 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=694938, jitterRate=-0.11634127795696259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:33:52,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731774832782Initializing all the Stores at 1731774832783 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774832783Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774832784 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774832784Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774832784Cleaning up temporary data from old regions at 1731774832807 (+23 ms)Region opened successfully at 1731774832823 (+16 ms)
2024-11-16T16:33:52,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:33:52,823 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:33:52,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:33:52,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:33:52,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:33:52,825 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:33:52,825 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774832823Disabling compacts and flushes for region at 1731774832823Disabling writes for close at 1731774832824 (+1 ms)Writing region close event to WAL at 1731774832825 (+1 ms)Closed at 1731774832825
2024-11-16T16:33:52,829 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:33:52,829 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:33:52,836 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:33:52,845 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:33:52,847 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34725, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:33:52,848 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:33:52,853 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39093 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,855 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39093 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,869 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c
2024-11-16T16:33:52,869 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36983
2024-11-16T16:33:52,869 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:33:52,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:33:52,879 DEBUG [RS:0;303fe01965e8:35937 {}] zookeeper.ZKUtil(111): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,880 WARN  [RS:0;303fe01965e8:35937 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:33:52,880 INFO  [RS:0;303fe01965e8:35937 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:33:52,880 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,883 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,35937,1731774831611]
2024-11-16T16:33:52,905 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:33:52,919 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:33:52,923 INFO  [RS:0;303fe01965e8:35937 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:33:52,923 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,924 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:33:52,930 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:33:52,931 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,931 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,931 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,932 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,932 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,932 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,932 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:33:52,933 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,933 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,933 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,933 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,933 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,934 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:33:52,934 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:33:52,934 DEBUG [RS:0;303fe01965e8:35937 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:33:52,935 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,935 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,935 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,935 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,936 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,936 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35937,1731774831611-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:33:52,953 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:33:52,955 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35937,1731774831611-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,955 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,955 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.Replication(171): 303fe01965e8,35937,1731774831611 started
2024-11-16T16:33:52,976 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:52,976 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,35937,1731774831611, RpcServer on 303fe01965e8/172.17.0.2:35937, sessionid=0x101447ed1480001
2024-11-16T16:33:52,977 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:33:52,978 DEBUG [RS:0;303fe01965e8:35937 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,978 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,35937,1731774831611'
2024-11-16T16:33:52,978 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:33:52,979 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:33:52,980 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:33:52,980 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:33:52,980 DEBUG [RS:0;303fe01965e8:35937 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,35937,1731774831611
2024-11-16T16:33:52,980 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,35937,1731774831611'
2024-11-16T16:33:52,981 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:33:52,981 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:33:52,982 DEBUG [RS:0;303fe01965e8:35937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:33:52,982 INFO  [RS:0;303fe01965e8:35937 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:33:52,982 INFO  [RS:0;303fe01965e8:35937 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:33:52,999 WARN  [303fe01965e8:39093 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:33:53,094 INFO  [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C35937%2C1731774831611, suffix=, logDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611, archiveDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs, maxLogs=32
2024-11-16T16:33:53,097 INFO  [RS:0;303fe01965e8:35937 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774833097
2024-11-16T16:33:53,106 INFO  [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774833097
2024-11-16T16:33:53,107 DEBUG [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:33:53,255 DEBUG [303fe01965e8:39093 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:33:53,268 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,35937,1731774831611
2024-11-16T16:33:53,273 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,35937,1731774831611, state=OPENING
2024-11-16T16:33:53,289 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:33:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:53,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:33:53,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:33:53,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:33:53,303 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:33:53,305 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,35937,1731774831611}]
2024-11-16T16:33:53,486 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:33:53,489 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40895, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:33:53,499 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:33:53,500 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:33:53,504 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C35937%2C1731774831611.meta, suffix=.meta, logDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611, archiveDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs, maxLogs=32
2024-11-16T16:33:53,506 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.meta.1731774833506.meta
2024-11-16T16:33:53,513 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.meta.1731774833506.meta
2024-11-16T16:33:53,515 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:33:53,516 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:33:53,518 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:33:53,520 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:33:53,524 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:33:53,528 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:33:53,529 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:53,529 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:33:53,530 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:33:53,532 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:33:53,534 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:33:53,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:53,535 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:53,535 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:33:53,537 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:33:53,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:53,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:53,538 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:33:53,539 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:33:53,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:53,540 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:53,541 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:33:53,542 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:33:53,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:53,543 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:33:53,543 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:33:53,544 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740
2024-11-16T16:33:53,547 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740
2024-11-16T16:33:53,549 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:33:53,550 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:33:53,551 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:33:53,553 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:33:53,555 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=727263, jitterRate=-0.0752381682395935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:33:53,555 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:33:53,556 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731774833530Writing region info on filesystem at 1731774833530Initializing all the Stores at 1731774833532 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774833532Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774833532Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774833532Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774833532Cleaning up temporary data from old regions at 1731774833550 (+18 ms)Running coprocessor post-open hooks at 1731774833555 (+5 ms)Region opened successfully at 1731774833556 (+1 ms)
2024-11-16T16:33:53,563 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731774833478
2024-11-16T16:33:53,574 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:33:53,575 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:33:53,577 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,35937,1731774831611
2024-11-16T16:33:53,579 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,35937,1731774831611, state=OPEN
2024-11-16T16:33:53,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:33:53,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:33:53,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:33:53,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:33:53,647 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,35937,1731774831611
2024-11-16T16:33:53,653 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:33:53,654 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,35937,1731774831611 in 344 msec
2024-11-16T16:33:53,662 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:33:53,662 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 819 msec
2024-11-16T16:33:53,664 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:33:53,664 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:33:53,681 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:33:53,682 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,35937,1731774831611, seqNum=-1]
2024-11-16T16:33:53,700 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:33:53,702 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50295, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:33:53,720 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0420 sec
2024-11-16T16:33:53,721 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731774833721, completionTime=-1
2024-11-16T16:33:53,723 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:33:53,724 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:33:53,748 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:33:53,748 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731774893748
2024-11-16T16:33:53,748 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731774953748
2024-11-16T16:33:53,748 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec
2024-11-16T16:33:53,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,753 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:39093, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,753 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,754 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,760 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:33:53,780 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.005sec
2024-11-16T16:33:53,782 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:33:53,783 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:33:53,784 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:33:53,784 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:33:53,784 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:33:53,785 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:33:53,786 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:33:53,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:33:53,796 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:33:53,797 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39093,1731774830835-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:33:53,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2961789e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:33:53,815 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-16T16:33:53,815 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-16T16:33:53,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,39093,-1 for getting cluster id
2024-11-16T16:33:53,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:33:53,828 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f53d255-655e-45fb-b61e-3fec7d6cedbd'
2024-11-16T16:33:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:33:53,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f53d255-655e-45fb-b61e-3fec7d6cedbd"

2024-11-16T16:33:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34757156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:33:53,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,39093,-1]
2024-11-16T16:33:53,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:33:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:33:53,839 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:33:53,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a039b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:33:53,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:33:53,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,35937,1731774831611, seqNum=-1]
2024-11-16T16:33:53,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:33:53,853 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:33:53,890 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,39093,1731774830835
2024-11-16T16:33:53,890 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:33:53,900 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:33:53,905 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T16:33:53,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,39093,1731774830835
2024-11-16T16:33:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51ccabd5
2024-11-16T16:33:53,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T16:33:53,921 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T16:33:53,924 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-16T16:33:53,924 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-16T16:33:53,930 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:33:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-16T16:33:53,945 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T16:33:53,948 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-16T16:33:53,948 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:53,951 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T16:33:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:33:53,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741835_1011 (size=389)
2024-11-16T16:33:53,996 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741835_1011 (size=389)
2024-11-16T16:33:53,999 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 671c524a21c4dd95c38ca641153081cd, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c
2024-11-16T16:33:54,008 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741836_1012 (size=72)
2024-11-16T16:33:54,009 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741836_1012 (size=72)
2024-11-16T16:33:54,009 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:54,010 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 671c524a21c4dd95c38ca641153081cd, disabling compactions & flushes
2024-11-16T16:33:54,010 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,010 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,010 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd. after waiting 0 ms
2024-11-16T16:33:54,010 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,010 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,010 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 671c524a21c4dd95c38ca641153081cd:
Waiting for close lock at 1731774834010Disabling compacts and flushes for region at 1731774834010Disabling writes for close at 1731774834010Writing region close event to WAL at 1731774834010Closed at 1731774834010
2024-11-16T16:33:54,012 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T16:33:54,016 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731774834012"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731774834012"}]},"ts":"1731774834012"}
2024-11-16T16:33:54,021 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T16:33:54,023 INFO  [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T16:33:54,026 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774834023"}]},"ts":"1731774834023"}
2024-11-16T16:33:54,030 INFO  [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-16T16:33:54,032 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=671c524a21c4dd95c38ca641153081cd, ASSIGN}]
2024-11-16T16:33:54,035 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=671c524a21c4dd95c38ca641153081cd, ASSIGN
2024-11-16T16:33:54,037 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=671c524a21c4dd95c38ca641153081cd, ASSIGN; state=OFFLINE, location=303fe01965e8,35937,1731774831611; forceNewPlan=false, retain=false
2024-11-16T16:33:54,188 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=671c524a21c4dd95c38ca641153081cd, regionState=OPENING, regionLocation=303fe01965e8,35937,1731774831611
2024-11-16T16:33:54,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=671c524a21c4dd95c38ca641153081cd, ASSIGN because future has completed
2024-11-16T16:33:54,195 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 671c524a21c4dd95c38ca641153081cd, server=303fe01965e8,35937,1731774831611}]
2024-11-16T16:33:54,364 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,364 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 671c524a21c4dd95c38ca641153081cd, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:33:54,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:33:54,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,368 INFO  [StoreOpener-671c524a21c4dd95c38ca641153081cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 671c524a21c4dd95c38ca641153081cd 
2024-11-16T16:33:54,371 INFO  [StoreOpener-671c524a21c4dd95c38ca641153081cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 671c524a21c4dd95c38ca641153081cd columnFamilyName info
2024-11-16T16:33:54,371 DEBUG [StoreOpener-671c524a21c4dd95c38ca641153081cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:33:54,372 INFO  [StoreOpener-671c524a21c4dd95c38ca641153081cd-1 {}] regionserver.HStore(327): Store=671c524a21c4dd95c38ca641153081cd/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:33:54,372 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,373 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,374 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,375 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,375 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,377 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,381 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:33:54,381 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 671c524a21c4dd95c38ca641153081cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=817915, jitterRate=0.0400337278842926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:33:54,382 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:33:54,383 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 671c524a21c4dd95c38ca641153081cd:
Running coprocessor pre-open hook at 1731774834366Writing region info on filesystem at 1731774834366Initializing all the Stores at 1731774834367 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774834367Cleaning up temporary data from old regions at 1731774834375 (+8 ms)Running coprocessor post-open hooks at 1731774834382 (+7 ms)Region opened successfully at 1731774834383 (+1 ms)
2024-11-16T16:33:54,385 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd., pid=6, masterSystemTime=1731774834351
2024-11-16T16:33:54,390 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,390 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:33:54,392 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=671c524a21c4dd95c38ca641153081cd, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,35937,1731774831611
2024-11-16T16:33:54,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 671c524a21c4dd95c38ca641153081cd, server=303fe01965e8,35937,1731774831611 because future has completed
2024-11-16T16:33:54,405 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T16:33:54,405 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 671c524a21c4dd95c38ca641153081cd, server=303fe01965e8,35937,1731774831611 in 206 msec
2024-11-16T16:33:54,409 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T16:33:54,409 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=671c524a21c4dd95c38ca641153081cd, ASSIGN in 373 msec
2024-11-16T16:33:54,411 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T16:33:54,411 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774834411"}]},"ts":"1731774834411"}
2024-11-16T16:33:54,414 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-16T16:33:54,416 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T16:33:54,420 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 481 msec
2024-11-16T16:33:59,048 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-16T16:33:59,095 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:33:59,097 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-16T16:34:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:34:01,190 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T16:34:01,194 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-16T16:34:01,194 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-16T16:34:01,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:34:01,197 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T16:34:01,198 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-16T16:34:01,198 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-16T16:34:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39093 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:34:04,011 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-16T16:34:04,016 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-16T16:34:04,024 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-16T16:34:04,025 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:34:04,026 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774844025
2024-11-16T16:34:04,036 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:04,036 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:04,036 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:04,036 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:04,036 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:04,037 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774833097 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774844025
2024-11-16T16:34:04,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44591:44591),(127.0.0.1/127.0.0.1:37775:37775)]
2024-11-16T16:34:04,039 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774833097 is not closed yet, will try archiving it next time
2024-11-16T16:34:04,040 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741833_1009 (size=451)
2024-11-16T16:34:04,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741833_1009 (size=451)
2024-11-16T16:34:04,043 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774833097 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774833097
2024-11-16T16:34:04,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd., hostname=303fe01965e8,35937,1731774831611, seqNum=2]
2024-11-16T16:34:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35937 {}] regionserver.HRegion(8855): Flush requested on 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:34:16,097 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 671c524a21c4dd95c38ca641153081cd 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:34:16,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/886cf57f45e043729b1fa3c41aa19235 is 1080, key is row0001/info:/1731774844051/Put/seqid=0
2024-11-16T16:34:16,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741838_1014 (size=12509)
2024-11-16T16:34:16,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741838_1014 (size=12509)
2024-11-16T16:34:16,182 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/886cf57f45e043729b1fa3c41aa19235
2024-11-16T16:34:16,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/886cf57f45e043729b1fa3c41aa19235 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235
2024-11-16T16:34:16,241 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235, entries=7, sequenceid=11, filesize=12.2 K
2024-11-16T16:34:16,248 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 150ms, sequenceid=11, compaction requested=false
2024-11-16T16:34:16,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 671c524a21c4dd95c38ca641153081cd:

2024-11-16T16:34:19,920 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:34:24,105 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774864105
2024-11-16T16:34:24,312 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:34:24,313 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:24,313 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:24,313 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:24,313 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:24,313 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:24,313 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774844025 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774864105
2024-11-16T16:34:24,314 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:34:24,314 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774844025 is not closed yet, will try archiving it next time
2024-11-16T16:34:24,316 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741837_1013 (size=12399)
2024-11-16T16:34:24,316 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741837_1013 (size=12399)
2024-11-16T16:34:24,518 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:26,722 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:28,926 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:31,129 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:31,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35937 {}] regionserver.HRegion(8855): Flush requested on 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:34:31,130 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 671c524a21c4dd95c38ca641153081cd 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:34:31,332 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:31,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/c50f62f404ab4bb9bb76fdf2550328c9 is 1080, key is row0008/info:/1731774858096/Put/seqid=0
2024-11-16T16:34:31,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741840_1016 (size=12509)
2024-11-16T16:34:31,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741840_1016 (size=12509)
2024-11-16T16:34:31,352 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/c50f62f404ab4bb9bb76fdf2550328c9
2024-11-16T16:34:31,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/c50f62f404ab4bb9bb76fdf2550328c9 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9
2024-11-16T16:34:31,374 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9, entries=7, sequenceid=21, filesize=12.2 K
2024-11-16T16:34:31,576 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:31,576 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 447ms, sequenceid=21, compaction requested=false
2024-11-16T16:34:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 671c524a21c4dd95c38ca641153081cd:

2024-11-16T16:34:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-16T16:34:31,576 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:34:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235 because midkey is the same as first or last row
2024-11-16T16:34:33,334 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:33,798 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-16T16:34:33,798 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-16T16:34:35,538 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 200 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:35,540 WARN  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:35,541 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C35937%2C1731774831611:(num 1731774864105) roll requested
2024-11-16T16:34:35,541 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774875541
2024-11-16T16:34:35,750 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:35,750 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:35,750 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:35,750 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:35,750 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:35,750 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:35,751 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774864105 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774875541
2024-11-16T16:34:35,752 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:34:35,752 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774864105 is not closed yet, will try archiving it next time
2024-11-16T16:34:35,752 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774844025 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774844025
2024-11-16T16:34:35,753 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741839_1015 (size=7739)
2024-11-16T16:34:35,754 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741839_1015 (size=7739)
2024-11-16T16:34:37,744 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:39,365 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 671c524a21c4dd95c38ca641153081cd, had cached 0 bytes from a total of 25018
2024-11-16T16:34:39,953 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 202 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:42,158 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:44,362 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:46,365 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-16T16:34:46,366 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774886366
2024-11-16T16:34:49,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:34:51,380 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5009 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:51,383 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5009 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:34:51,383 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C35937%2C1731774831611:(num 1731774886366) roll requested
2024-11-16T16:34:51,383 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:51,383 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:51,383 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:51,383 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:51,383 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:34:51,384 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774875541 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774886366
2024-11-16T16:34:51,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44591:44591),(127.0.0.1/127.0.0.1:37775:37775)]
2024-11-16T16:34:51,385 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774875541 is not closed yet, will try archiving it next time
2024-11-16T16:34:51,386 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774891385
2024-11-16T16:34:51,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741841_1017 (size=4753)
2024-11-16T16:34:51,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741841_1017 (size=4753)
2024-11-16T16:34:56,390 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:34:56,390 WARN  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:34:56,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35937 {}] regionserver.HRegion(8855): Flush requested on 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:34:56,390 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 671c524a21c4dd95c38ca641153081cd 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:34:56,401 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:34:56,401 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:34:58,391 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-16T16:35:01,393 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:35:01,393 WARN  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK], DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK]]
2024-11-16T16:35:01,393 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:01,394 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:01,394 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:01,394 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:01,394 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:01,395 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774886366 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385
2024-11-16T16:35:01,396 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:35:01,397 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774886366 is not closed yet, will try archiving it next time
2024-11-16T16:35:01,397 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C35937%2C1731774831611:(num 1731774891385) roll requested
2024-11-16T16:35:01,398 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774901397
2024-11-16T16:35:01,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741842_1018 (size=1569)
2024-11-16T16:35:01,401 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741842_1018 (size=1569)
2024-11-16T16:35:01,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/a6455b4727804b9fb1414e37c1f69dc9 is 1080, key is row0015/info:/1731774873132/Put/seqid=0
2024-11-16T16:35:01,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741844_1020 (size=12509)
2024-11-16T16:35:01,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741844_1020 (size=12509)
2024-11-16T16:35:01,411 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/a6455b4727804b9fb1414e37c1f69dc9
2024-11-16T16:35:01,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/a6455b4727804b9fb1414e37c1f69dc9 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9
2024-11-16T16:35:01,430 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9, entries=7, sequenceid=31, filesize=12.2 K
2024-11-16T16:35:06,407 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:35:06,407 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:35:06,432 INFO  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:35:06,432 WARN  [FSHLog-0-hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c-prefix:303fe01965e8,35937,1731774831611 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39383,DS-db944295-0531-4e0c-baa5-a6c1f976c343,DISK], DatanodeInfoWithStorage[127.0.0.1:45495,DS-a01e59ff-95a4-4449-a034-2687d1d4f202,DISK]]
2024-11-16T16:35:06,432 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,432 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 10042ms, sequenceid=31, compaction requested=true
2024-11-16T16:35:06,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 671c524a21c4dd95c38ca641153081cd:

2024-11-16T16:35:06,432 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,433 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,433 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,433 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-16T16:35:06,433 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,433 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:06,433 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235 because midkey is the same as first or last row
2024-11-16T16:35:06,433 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774901397
2024-11-16T16:35:06,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741843_1019 (size=438)
2024-11-16T16:35:06,436 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741843_1019 (size=438)
2024-11-16T16:35:06,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 671c524a21c4dd95c38ca641153081cd:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:35:06,439 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44591:44591),(127.0.0.1/127.0.0.1:37775:37775)]
2024-11-16T16:35:06,439 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 is not closed yet, will try archiving it next time
2024-11-16T16:35:06,440 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774864105 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774864105
2024-11-16T16:35:06,440 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C35937%2C1731774831611:(num 1731774906440) roll requested
2024-11-16T16:35:06,440 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774906440
2024-11-16T16:35:06,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:06,442 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:35:06,442 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774875541 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774875541
2024-11-16T16:35:06,444 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774886366 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774886366
2024-11-16T16:35:06,448 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:35:06,450 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HStore(1541): 671c524a21c4dd95c38ca641153081cd/info is initiating minor compaction (all files)
2024-11-16T16:35:06,451 INFO  [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 671c524a21c4dd95c38ca641153081cd/info in TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:06,451 INFO  [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9] into tmpdir=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp, totalSize=36.6 K
2024-11-16T16:35:06,453 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886cf57f45e043729b1fa3c41aa19235, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731774844051
2024-11-16T16:35:06,454 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] compactions.Compactor(225): Compacting c50f62f404ab4bb9bb76fdf2550328c9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731774858096
2024-11-16T16:35:06,455 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6455b4727804b9fb1414e37c1f69dc9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731774873132
2024-11-16T16:35:06,463 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,463 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,463 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,463 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,464 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,464 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774901397 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774906440
2024-11-16T16:35:06,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741845_1021 (size=93)
2024-11-16T16:35:06,466 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741845_1021 (size=93)
2024-11-16T16:35:06,467 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 is not closed yet, will try archiving it next time
2024-11-16T16:35:06,467 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774901397 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774901397
2024-11-16T16:35:06,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:35:06,473 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 is not closed yet, will try archiving it next time
2024-11-16T16:35:06,474 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35937%2C1731774831611.1731774906473
2024-11-16T16:35:06,486 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,486 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,486 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,486 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,486 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:06,487 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774906440 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774906473
2024-11-16T16:35:06,489 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741846_1022 (size=1258)
2024-11-16T16:35:06,491 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741846_1022 (size=1258)
2024-11-16T16:35:06,492 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 is not closed yet, will try archiving it next time
2024-11-16T16:35:06,500 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37775:37775),(127.0.0.1/127.0.0.1:44591:44591)]
2024-11-16T16:35:06,501 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 is not closed yet, will try archiving it next time
2024-11-16T16:35:06,509 INFO  [RS:0;303fe01965e8:35937-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 671c524a21c4dd95c38ca641153081cd#info#compaction#3 average throughput is 7.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:35:06,511 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/c230fe9dcc3343098ecc4e5da1a7701e is 1080, key is row0001/info:/1731774844051/Put/seqid=0
2024-11-16T16:35:06,524 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741848_1024 (size=27710)
2024-11-16T16:35:06,525 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741848_1024 (size=27710)
2024-11-16T16:35:06,540 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/c230fe9dcc3343098ecc4e5da1a7701e as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c230fe9dcc3343098ecc4e5da1a7701e
2024-11-16T16:35:06,565 INFO  [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 671c524a21c4dd95c38ca641153081cd/info of 671c524a21c4dd95c38ca641153081cd into c230fe9dcc3343098ecc4e5da1a7701e(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:35:06,565 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 671c524a21c4dd95c38ca641153081cd:

2024-11-16T16:35:06,567 INFO  [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd., storeName=671c524a21c4dd95c38ca641153081cd/info, priority=13, startTime=1731774906434; duration=0sec
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c230fe9dcc3343098ecc4e5da1a7701e because midkey is the same as first or last row
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:06,568 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c230fe9dcc3343098ecc4e5da1a7701e because midkey is the same as first or last row
2024-11-16T16:35:06,569 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-16T16:35:06,569 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:06,569 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c230fe9dcc3343098ecc4e5da1a7701e because midkey is the same as first or last row
2024-11-16T16:35:06,569 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:06,569 DEBUG [RS:0;303fe01965e8:35937-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 671c524a21c4dd95c38ca641153081cd:info
2024-11-16T16:35:06,838 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/WALs/303fe01965e8,35937,1731774831611/303fe01965e8%2C35937%2C1731774831611.1731774891385 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs/303fe01965e8%2C35937%2C1731774831611.1731774891385
2024-11-16T16:35:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35937 {}] regionserver.HRegion(8855): Flush requested on 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:35:18,508 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 671c524a21c4dd95c38ca641153081cd 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:35:18,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/6010fc313c7d41a886c56415b191b123 is 1080, key is row0022/info:/1731774906475/Put/seqid=0
2024-11-16T16:35:18,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741849_1025 (size=12509)
2024-11-16T16:35:18,529 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741849_1025 (size=12509)
2024-11-16T16:35:18,530 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/6010fc313c7d41a886c56415b191b123
2024-11-16T16:35:18,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/6010fc313c7d41a886c56415b191b123 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/6010fc313c7d41a886c56415b191b123
2024-11-16T16:35:18,548 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/6010fc313c7d41a886c56415b191b123, entries=7, sequenceid=42, filesize=12.2 K
2024-11-16T16:35:18,549 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 42ms, sequenceid=42, compaction requested=false
2024-11-16T16:35:18,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 671c524a21c4dd95c38ca641153081cd:

2024-11-16T16:35:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-16T16:35:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c230fe9dcc3343098ecc4e5da1a7701e because midkey is the same as first or last row
2024-11-16T16:35:19,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:35:24,366 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 671c524a21c4dd95c38ca641153081cd, had cached 0 bytes from a total of 40219
2024-11-16T16:35:26,522 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:35:26,522 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:35:26,523 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:26,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:26,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:26,530 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:35:26,530 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:35:26,531 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=368815740, stopped=false
2024-11-16T16:35:26,531 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,39093,1731774830835
2024-11-16T16:35:26,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:26,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:26,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:26,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:26,618 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:35:26,618 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:35:26,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:26,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:26,619 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:26,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:26,620 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,35937,1731774831611' *****
2024-11-16T16:35:26,620 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:35:26,621 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:35:26,621 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:35:26,621 INFO  [RS:0;303fe01965e8:35937 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:35:26,622 INFO  [RS:0;303fe01965e8:35937 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:35:26,622 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(3091): Received CLOSE for 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:35:26,623 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,35937,1731774831611
2024-11-16T16:35:26,623 INFO  [RS:0;303fe01965e8:35937 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:35:26,623 INFO  [RS:0;303fe01965e8:35937 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:35937.
2024-11-16T16:35:26,623 DEBUG [RS:0;303fe01965e8:35937 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:26,623 DEBUG [RS:0;303fe01965e8:35937 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:26,624 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 671c524a21c4dd95c38ca641153081cd, disabling compactions & flushes
2024-11-16T16:35:26,624 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:26,624 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:35:26,624 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:26,624 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:35:26,624 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:35:26,624 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd. after waiting 0 ms
2024-11-16T16:35:26,624 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:26,624 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:35:26,624 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 671c524a21c4dd95c38ca641153081cd 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-16T16:35:26,625 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-16T16:35:26,625 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:35:26,625 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 671c524a21c4dd95c38ca641153081cd=TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.}
2024-11-16T16:35:26,625 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:35:26,625 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:35:26,625 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:35:26,625 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:35:26,625 DEBUG [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 671c524a21c4dd95c38ca641153081cd
2024-11-16T16:35:26,625 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-16T16:35:26,630 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/1da757820f8f4af691d8d231521b8295 is 1080, key is row0029/info:/1731774920510/Put/seqid=0
2024-11-16T16:35:26,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741850_1026 (size=8193)
2024-11-16T16:35:26,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741850_1026 (size=8193)
2024-11-16T16:35:26,637 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/1da757820f8f4af691d8d231521b8295
2024-11-16T16:35:26,648 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/.tmp/info/1da757820f8f4af691d8d231521b8295 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/1da757820f8f4af691d8d231521b8295
2024-11-16T16:35:26,654 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/info/0fb7a89f06474ff6b50c7e84334a1f2e is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd./info:regioninfo/1731774834392/Put/seqid=0
2024-11-16T16:35:26,662 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/1da757820f8f4af691d8d231521b8295, entries=3, sequenceid=48, filesize=8.0 K
2024-11-16T16:35:26,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741851_1027 (size=7016)
2024-11-16T16:35:26,663 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741851_1027 (size=7016)
2024-11-16T16:35:26,664 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 40ms, sequenceid=48, compaction requested=true
2024-11-16T16:35:26,665 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9] to archive
2024-11-16T16:35:26,668 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/info/0fb7a89f06474ff6b50c7e84334a1f2e
2024-11-16T16:35:26,669 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:35:26,673 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/886cf57f45e043729b1fa3c41aa19235
2024-11-16T16:35:26,675 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/c50f62f404ab4bb9bb76fdf2550328c9
2024-11-16T16:35:26,679 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9 to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/archive/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/info/a6455b4727804b9fb1414e37c1f69dc9
2024-11-16T16:35:26,691 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=303fe01965e8:39093 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-16T16:35:26,695 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [886cf57f45e043729b1fa3c41aa19235=12509, c50f62f404ab4bb9bb76fdf2550328c9=12509, a6455b4727804b9fb1414e37c1f69dc9=12509]
2024-11-16T16:35:26,696 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/ns/08fd4678fddb43f893057b0a37ed146a is 43, key is default/ns:d/1731774833705/Put/seqid=0
2024-11-16T16:35:26,701 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/default/TestLogRolling-testSlowSyncLogRolling/671c524a21c4dd95c38ca641153081cd/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-16T16:35:26,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741852_1028 (size=5153)
2024-11-16T16:35:26,703 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741852_1028 (size=5153)
2024-11-16T16:35:26,703 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/ns/08fd4678fddb43f893057b0a37ed146a
2024-11-16T16:35:26,704 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:26,704 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 671c524a21c4dd95c38ca641153081cd:
Waiting for close lock at 1731774926623Running coprocessor pre-close hooks at 1731774926623Disabling compacts and flushes for region at 1731774926623Disabling writes for close at 1731774926624 (+1 ms)Obtaining lock to block concurrent updates at 1731774926624Preparing flush snapshotting stores in 671c524a21c4dd95c38ca641153081cd at 1731774926624Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731774926625 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd. at 1731774926626 (+1 ms)Flushing 671c524a21c4dd95c38ca641153081cd/info: creating writer at 1731774926626Flushing 671c524a21c4dd95c38ca641153081cd/info: appending metadata at 1731774926630 (+4 ms)Flushing 671c524a21c4dd95c38ca641153081cd/info: closing flushed file at 1731774926630Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ddbcfe0: reopening flushed file at 1731774926646 (+16 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 671c524a21c4dd95c38ca641153081cd in 40ms, sequenceid=48, compaction requested=true at 1731774926664 (+18 ms)Writing region close event to WAL at 1731774926696 (+32 ms)Running coprocessor post-close hooks at 1731774926702 (+6 ms)Closed at 1731774926704 (+2 ms)
2024-11-16T16:35:26,705 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731774833923.671c524a21c4dd95c38ca641153081cd.
2024-11-16T16:35:26,726 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/table/eb5b4ff83cf345b3a7bde1a3cf907a61 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731774834411/Put/seqid=0
2024-11-16T16:35:26,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741853_1029 (size=5396)
2024-11-16T16:35:26,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741853_1029 (size=5396)
2024-11-16T16:35:26,733 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/table/eb5b4ff83cf345b3a7bde1a3cf907a61
2024-11-16T16:35:26,740 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/info/0fb7a89f06474ff6b50c7e84334a1f2e as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/info/0fb7a89f06474ff6b50c7e84334a1f2e
2024-11-16T16:35:26,749 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/info/0fb7a89f06474ff6b50c7e84334a1f2e, entries=10, sequenceid=11, filesize=6.9 K
2024-11-16T16:35:26,750 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/ns/08fd4678fddb43f893057b0a37ed146a as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/ns/08fd4678fddb43f893057b0a37ed146a
2024-11-16T16:35:26,759 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/ns/08fd4678fddb43f893057b0a37ed146a, entries=2, sequenceid=11, filesize=5.0 K
2024-11-16T16:35:26,760 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/.tmp/table/eb5b4ff83cf345b3a7bde1a3cf907a61 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/table/eb5b4ff83cf345b3a7bde1a3cf907a61
2024-11-16T16:35:26,769 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/table/eb5b4ff83cf345b3a7bde1a3cf907a61, entries=2, sequenceid=11, filesize=5.3 K
2024-11-16T16:35:26,771 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 146ms, sequenceid=11, compaction requested=false
2024-11-16T16:35:26,778 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-16T16:35:26,778 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:35:26,779 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:26,779 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774926625Running coprocessor pre-close hooks at 1731774926625Disabling compacts and flushes for region at 1731774926625Disabling writes for close at 1731774926625Obtaining lock to block concurrent updates at 1731774926625Preparing flush snapshotting stores in 1588230740 at 1731774926625Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731774926626 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731774926627 (+1 ms)Flushing 1588230740/info: creating writer at 1731774926627Flushing 1588230740/info: appending metadata at 1731774926653 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731774926653Flushing 1588230740/ns: creating writer at 1731774926678 (+25 ms)Flushing 1588230740/ns: appending metadata at 1731774926696 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731774926696Flushing 1588230740/table: creating writer at 1731774926711 (+15 ms)Flushing 1588230740/table: appending metadata at 1731774926726 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731774926726Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7499498e: reopening flushed file at 1731774926739 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@488845f0: reopening flushed file at 1731774926749 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@420bc30f: reopening flushed file at 1731774926759 (+10 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 146ms, sequenceid=11, compaction requested=false at 1731774926771 (+12 ms)Writing region close event to WAL at 1731774926773 (+2 ms)Running coprocessor post-close hooks at 1731774926778 (+5 ms)Closed at 1731774926779 (+1 ms)
2024-11-16T16:35:26,779 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:26,826 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,35937,1731774831611; all regions closed.
2024-11-16T16:35:26,827 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:26,827 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:26,828 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:26,828 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:26,828 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:26,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741834_1010 (size=3066)
2024-11-16T16:35:26,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741834_1010 (size=3066)
2024-11-16T16:35:26,937 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T16:35:26,938 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T16:35:26,940 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:35:27,236 DEBUG [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs
2024-11-16T16:35:27,236 INFO  [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C35937%2C1731774831611.meta:.meta(num 1731774833506)
2024-11-16T16:35:27,237 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,237 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,237 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,237 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,237 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741847_1023 (size=12695)
2024-11-16T16:35:27,240 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741847_1023 (size=12695)
2024-11-16T16:35:27,245 DEBUG [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/oldWALs
2024-11-16T16:35:27,245 INFO  [RS:0;303fe01965e8:35937 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C35937%2C1731774831611:(num 1731774906473)
2024-11-16T16:35:27,245 DEBUG [RS:0;303fe01965e8:35937 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:27,245 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:35:27,246 INFO  [RS:0;303fe01965e8:35937 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:35:27,246 INFO  [RS:0;303fe01965e8:35937 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T16:35:27,246 INFO  [RS:0;303fe01965e8:35937 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:35:27,246 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:35:27,247 INFO  [RS:0;303fe01965e8:35937 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35937
2024-11-16T16:35:27,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,35937,1731774831611
2024-11-16T16:35:27,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:35:27,302 INFO  [RS:0;303fe01965e8:35937 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:35:27,304 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,35937,1731774831611]
2024-11-16T16:35:27,327 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,35937,1731774831611 already deleted, retry=false
2024-11-16T16:35:27,328 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,35937,1731774831611 expired; onlineServers=0
2024-11-16T16:35:27,328 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,39093,1731774830835' *****
2024-11-16T16:35:27,328 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:35:27,329 INFO  [M:0;303fe01965e8:39093 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:35:27,329 INFO  [M:0;303fe01965e8:39093 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:35:27,329 DEBUG [M:0;303fe01965e8:39093 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:35:27,330 DEBUG [M:0;303fe01965e8:39093 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:35:27,330 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:35:27,330 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774832761 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774832761,5,FailOnTimeoutGroup]
2024-11-16T16:35:27,330 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774832760 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774832760,5,FailOnTimeoutGroup]
2024-11-16T16:35:27,331 INFO  [M:0;303fe01965e8:39093 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:35:27,332 INFO  [M:0;303fe01965e8:39093 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:35:27,332 DEBUG [M:0;303fe01965e8:39093 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:35:27,333 INFO  [M:0;303fe01965e8:39093 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:35:27,333 INFO  [M:0;303fe01965e8:39093 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:35:27,335 INFO  [M:0;303fe01965e8:39093 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:35:27,335 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:35:27,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:35:27,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:27,343 DEBUG [M:0;303fe01965e8:39093 {}] zookeeper.ZKUtil(347): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:35:27,343 WARN  [M:0;303fe01965e8:39093 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:35:27,345 INFO  [M:0;303fe01965e8:39093 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/.lastflushedseqids
2024-11-16T16:35:27,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741854_1030 (size=130)
2024-11-16T16:35:27,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741854_1030 (size=130)
2024-11-16T16:35:27,357 INFO  [M:0;303fe01965e8:39093 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:35:27,357 INFO  [M:0;303fe01965e8:39093 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:35:27,357 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:35:27,357 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:27,357 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:27,357 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:35:27,357 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:27,357 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-11-16T16:35:27,380 DEBUG [M:0;303fe01965e8:39093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4858745416416283d0d28124bc625f is 82, key is hbase:meta,,1/info:regioninfo/1731774833576/Put/seqid=0
2024-11-16T16:35:27,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741855_1031 (size=5672)
2024-11-16T16:35:27,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741855_1031 (size=5672)
2024-11-16T16:35:27,387 INFO  [M:0;303fe01965e8:39093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4858745416416283d0d28124bc625f
2024-11-16T16:35:27,409 DEBUG [M:0;303fe01965e8:39093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/702f221b49f24ac8881ca907d5a6c286 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731774834419/Put/seqid=0
2024-11-16T16:35:27,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741856_1032 (size=6247)
2024-11-16T16:35:27,416 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741856_1032 (size=6247)
2024-11-16T16:35:27,416 INFO  [M:0;303fe01965e8:39093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/702f221b49f24ac8881ca907d5a6c286
2024-11-16T16:35:27,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:27,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35937-0x101447ed1480001, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:27,418 INFO  [RS:0;303fe01965e8:35937 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:35:27,418 INFO  [RS:0;303fe01965e8:35937 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,35937,1731774831611; zookeeper connection closed.
2024-11-16T16:35:27,418 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@606f7396 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@606f7396
2024-11-16T16:35:27,419 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:35:27,423 INFO  [M:0;303fe01965e8:39093 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 702f221b49f24ac8881ca907d5a6c286
2024-11-16T16:35:27,438 DEBUG [M:0;303fe01965e8:39093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9333d31d2a374c9ca0a5c6041709fbdd is 69, key is 303fe01965e8,35937,1731774831611/rs:state/1731774832857/Put/seqid=0
2024-11-16T16:35:27,443 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741857_1033 (size=5156)
2024-11-16T16:35:27,444 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741857_1033 (size=5156)
2024-11-16T16:35:27,445 INFO  [M:0;303fe01965e8:39093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9333d31d2a374c9ca0a5c6041709fbdd
2024-11-16T16:35:27,476 DEBUG [M:0;303fe01965e8:39093 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/968346ab5b664f63ab3679e353680f77 is 52, key is load_balancer_on/state:d/1731774833896/Put/seqid=0
2024-11-16T16:35:27,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741858_1034 (size=5056)
2024-11-16T16:35:27,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741858_1034 (size=5056)
2024-11-16T16:35:27,483 INFO  [M:0;303fe01965e8:39093 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/968346ab5b664f63ab3679e353680f77
2024-11-16T16:35:27,491 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8b4858745416416283d0d28124bc625f as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b4858745416416283d0d28124bc625f
2024-11-16T16:35:27,498 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8b4858745416416283d0d28124bc625f, entries=8, sequenceid=59, filesize=5.5 K
2024-11-16T16:35:27,500 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/702f221b49f24ac8881ca907d5a6c286 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/702f221b49f24ac8881ca907d5a6c286
2024-11-16T16:35:27,509 INFO  [M:0;303fe01965e8:39093 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 702f221b49f24ac8881ca907d5a6c286
2024-11-16T16:35:27,509 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/702f221b49f24ac8881ca907d5a6c286, entries=6, sequenceid=59, filesize=6.1 K
2024-11-16T16:35:27,510 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9333d31d2a374c9ca0a5c6041709fbdd as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9333d31d2a374c9ca0a5c6041709fbdd
2024-11-16T16:35:27,518 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9333d31d2a374c9ca0a5c6041709fbdd, entries=1, sequenceid=59, filesize=5.0 K
2024-11-16T16:35:27,520 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/968346ab5b664f63ab3679e353680f77 as hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/968346ab5b664f63ab3679e353680f77
2024-11-16T16:35:27,527 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/968346ab5b664f63ab3679e353680f77, entries=1, sequenceid=59, filesize=4.9 K
2024-11-16T16:35:27,528 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=59, compaction requested=false
2024-11-16T16:35:27,530 INFO  [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:27,530 DEBUG [M:0;303fe01965e8:39093 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774927357Disabling compacts and flushes for region at 1731774927357Disabling writes for close at 1731774927357Obtaining lock to block concurrent updates at 1731774927357Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731774927357Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731774927358 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731774927359 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731774927359Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731774927379 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731774927379Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731774927393 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731774927408 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731774927408Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731774927423 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731774927437 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731774927437Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731774927454 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731774927476 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731774927476Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5469e7c2: reopening flushed file at 1731774927490 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14375f1: reopening flushed file at 1731774927499 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3391fbc3: reopening flushed file at 1731774927509 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a9da9a6: reopening flushed file at 1731774927518 (+9 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=59, compaction requested=false at 1731774927528 (+10 ms)Writing region close event to WAL at 1731774927530 (+2 ms)Closed at 1731774927530
2024-11-16T16:35:27,531 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,531 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,531 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,531 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,531 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:27,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39383 is added to blk_1073741830_1006 (size=27973)
2024-11-16T16:35:27,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45495 is added to blk_1073741830_1006 (size=27973)
2024-11-16T16:35:27,535 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:35:27,535 INFO  [M:0;303fe01965e8:39093 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:35:27,535 INFO  [M:0;303fe01965e8:39093 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39093
2024-11-16T16:35:27,535 INFO  [M:0;303fe01965e8:39093 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:35:27,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:27,648 INFO  [M:0;303fe01965e8:39093 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:35:27,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39093-0x101447ed1480000, quorum=127.0.0.1:51607, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:27,655 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6c963ecd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:27,658 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a10aed{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:27,658 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:27,659 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25ca9bb3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:27,659 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ff5148a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:27,663 WARN  [BP-908077221-172.17.0.2-1731774826254 heartbeating to localhost/127.0.0.1:36983 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:27,663 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:27,663 WARN  [BP-908077221-172.17.0.2-1731774826254 heartbeating to localhost/127.0.0.1:36983 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-908077221-172.17.0.2-1731774826254 (Datanode Uuid 01a12fe5-d614-411c-8468-38ddaa27a747) service to localhost/127.0.0.1:36983
2024-11-16T16:35:27,663 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:27,664 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data3/current/BP-908077221-172.17.0.2-1731774826254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:27,665 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data4/current/BP-908077221-172.17.0.2-1731774826254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:27,665 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:27,668 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f93babe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:27,669 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@737d6c99{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:27,669 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:27,669 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7893eb07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:27,669 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3305dd74{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:27,671 WARN  [BP-908077221-172.17.0.2-1731774826254 heartbeating to localhost/127.0.0.1:36983 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:27,671 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:27,671 WARN  [BP-908077221-172.17.0.2-1731774826254 heartbeating to localhost/127.0.0.1:36983 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-908077221-172.17.0.2-1731774826254 (Datanode Uuid c27172ad-74de-4f82-ab3b-bf7b53f0c1f6) service to localhost/127.0.0.1:36983
2024-11-16T16:35:27,671 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:27,672 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data1/current/BP-908077221-172.17.0.2-1731774826254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:27,672 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/cluster_1fb2d9c3-25fa-49ac-34f7-a44294e438c8/data/data2/current/BP-908077221-172.17.0.2-1731774826254 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:27,673 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:27,686 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6de997b9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:35:27,687 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a0da00a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:27,687 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:27,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@380b8195{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:27,688 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3aee6cb7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:27,699 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:35:27,739 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:35:27,749 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12)
Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@5e1d09c6
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36983
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36983
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36983
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36983 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36983
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/303fe01965e8:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:36983
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36983 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/303fe01965e8:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36983 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: master/303fe01965e8:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-6
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: ForkJoinPool-2-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=4723 (was 5081)
2024-11-16T16:35:27,757 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=4720
2024-11-16T16:35:27,757 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.log.dir so I do NOT create it in target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/7d2b3a78-c685-6729-a60b-89bd5295d51c/hadoop.tmp.dir so I do NOT create it in target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b, deleteOnExit=true
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/test.cache.data in system properties and HBase conf
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:35:27,758 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:35:27,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:35:27,759 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:35:27,760 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:35:27,775 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:35:28,125 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:28,131 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:28,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:28,136 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:28,136 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:35:28,137 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:28,137 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5686c999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:28,138 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69fa0c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:28,260 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bc93f4{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/java.io.tmpdir/jetty-localhost-42381-hadoop-hdfs-3_4_1-tests_jar-_-any-7497088961642014905/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:35:28,261 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ec80534{HTTP/1.1, (http/1.1)}{localhost:42381}
2024-11-16T16:35:28,261 INFO  [Time-limited test {}] server.Server(415): Started @104037ms
2024-11-16T16:35:28,274 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:35:28,525 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:28,528 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:28,529 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:28,529 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:28,529 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:35:28,530 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5357e169{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:28,530 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39fb5954{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:28,624 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@77ff58b7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/java.io.tmpdir/jetty-localhost-39125-hadoop-hdfs-3_4_1-tests_jar-_-any-7316301972501990324/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:28,625 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9d8d6d6{HTTP/1.1, (http/1.1)}{localhost:39125}
2024-11-16T16:35:28,625 INFO  [Time-limited test {}] server.Server(415): Started @104401ms
2024-11-16T16:35:28,626 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:28,659 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:28,663 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:28,664 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:28,664 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:28,664 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:35:28,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7130e643{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:28,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a17cc25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:28,759 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a329c51{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/java.io.tmpdir/jetty-localhost-37949-hadoop-hdfs-3_4_1-tests_jar-_-any-8339737826739539262/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:28,759 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d8729f1{HTTP/1.1, (http/1.1)}{localhost:37949}
2024-11-16T16:35:28,759 INFO  [Time-limited test {}] server.Server(415): Started @104535ms
2024-11-16T16:35:28,761 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:29,631 WARN  [Thread-454 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data2/current/BP-353393911-172.17.0.2-1731774927787/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:29,631 WARN  [Thread-453 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data1/current/BP-353393911-172.17.0.2-1731774927787/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:29,653 WARN  [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:29,656 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd951c5d4fa67ae67 with lease ID 0x2b3fa3395778f777: Processing first storage report for DS-5d7dab24-12f5-406e-ab05-aba12ccd3d8e from datanode DatanodeRegistration(127.0.0.1:44815, datanodeUuid=0c31588d-7536-490d-afab-1ab1bcaad24d, infoPort=35353, infoSecurePort=0, ipcPort=34869, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787)
2024-11-16T16:35:29,656 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd951c5d4fa67ae67 with lease ID 0x2b3fa3395778f777: from storage DS-5d7dab24-12f5-406e-ab05-aba12ccd3d8e node DatanodeRegistration(127.0.0.1:44815, datanodeUuid=0c31588d-7536-490d-afab-1ab1bcaad24d, infoPort=35353, infoSecurePort=0, ipcPort=34869, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:29,657 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd951c5d4fa67ae67 with lease ID 0x2b3fa3395778f777: Processing first storage report for DS-d068024a-7b82-442f-8f68-22d5bb44151f from datanode DatanodeRegistration(127.0.0.1:44815, datanodeUuid=0c31588d-7536-490d-afab-1ab1bcaad24d, infoPort=35353, infoSecurePort=0, ipcPort=34869, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787)
2024-11-16T16:35:29,657 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd951c5d4fa67ae67 with lease ID 0x2b3fa3395778f777: from storage DS-d068024a-7b82-442f-8f68-22d5bb44151f node DatanodeRegistration(127.0.0.1:44815, datanodeUuid=0c31588d-7536-490d-afab-1ab1bcaad24d, infoPort=35353, infoSecurePort=0, ipcPort=34869, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:29,789 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data3/current/BP-353393911-172.17.0.2-1731774927787/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:29,789 WARN  [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data4/current/BP-353393911-172.17.0.2-1731774927787/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:29,804 WARN  [Thread-440 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:29,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b9ac74bed8b5987 with lease ID 0x2b3fa3395778f778: Processing first storage report for DS-5bce1f9f-b8db-4ef3-974e-b8993b1043ca from datanode DatanodeRegistration(127.0.0.1:36825, datanodeUuid=15741f7f-3d4e-4e3d-a023-c72960fb091b, infoPort=36287, infoSecurePort=0, ipcPort=42683, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787)
2024-11-16T16:35:29,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b9ac74bed8b5987 with lease ID 0x2b3fa3395778f778: from storage DS-5bce1f9f-b8db-4ef3-974e-b8993b1043ca node DatanodeRegistration(127.0.0.1:36825, datanodeUuid=15741f7f-3d4e-4e3d-a023-c72960fb091b, infoPort=36287, infoSecurePort=0, ipcPort=42683, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:29,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b9ac74bed8b5987 with lease ID 0x2b3fa3395778f778: Processing first storage report for DS-a35d47f6-a133-462e-b1bd-52da00c396dc from datanode DatanodeRegistration(127.0.0.1:36825, datanodeUuid=15741f7f-3d4e-4e3d-a023-c72960fb091b, infoPort=36287, infoSecurePort=0, ipcPort=42683, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787)
2024-11-16T16:35:29,806 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b9ac74bed8b5987 with lease ID 0x2b3fa3395778f778: from storage DS-a35d47f6-a133-462e-b1bd-52da00c396dc node DatanodeRegistration(127.0.0.1:36825, datanodeUuid=15741f7f-3d4e-4e3d-a023-c72960fb091b, infoPort=36287, infoSecurePort=0, ipcPort=42683, storageInfo=lv=-57;cid=testClusterID;nsid=1756020480;c=1731774927787), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:29,903 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab
2024-11-16T16:35:29,906 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/zookeeper_0, clientPort=57432, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:35:29,907 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57432
2024-11-16T16:35:29,907 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:29,909 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:29,919 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:35:29,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:35:29,921 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba with version=8
2024-11-16T16:35:29,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:35:29,923 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:35:29,923 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:35:29,924 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:35:29,925 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35499
2024-11-16T16:35:29,927 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35499 connecting to ZooKeeper ensemble=127.0.0.1:57432
2024-11-16T16:35:30,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:354990x0, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:35:30,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35499-0x1014480579d0000 connected
2024-11-16T16:35:30,216 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:30,221 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:30,225 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:30,225 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba, hbase.cluster.distributed=false
2024-11-16T16:35:30,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:35:30,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35499
2024-11-16T16:35:30,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35499
2024-11-16T16:35:30,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35499
2024-11-16T16:35:30,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35499
2024-11-16T16:35:30,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35499
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:35:30,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:35:30,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33199
2024-11-16T16:35:30,250 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33199 connecting to ZooKeeper ensemble=127.0.0.1:57432
2024-11-16T16:35:30,251 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:30,254 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:30,267 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331990x0, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:35:30,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33199-0x1014480579d0001 connected
2024-11-16T16:35:30,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:30,268 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:35:30,269 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:35:30,270 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:35:30,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:35:30,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33199
2024-11-16T16:35:30,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33199
2024-11-16T16:35:30,272 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33199
2024-11-16T16:35:30,273 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33199
2024-11-16T16:35:30,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33199
2024-11-16T16:35:30,285 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:35499
2024-11-16T16:35:30,285 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,295 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:30,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:30,296 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,306 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:35:30,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,306 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,307 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:35:30,308 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,35499,1731774929923 from backup master directory
2024-11-16T16:35:30,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,320 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:30,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:30,320 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:35:30,320 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,328 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/hbase.id] with ID: 982efa4d-21e9-479e-8f52-34456a664fc5
2024-11-16T16:35:30,328 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/.tmp/hbase.id
2024-11-16T16:35:30,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:35:30,337 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:35:30,338 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/.tmp/hbase.id]:[hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/hbase.id]
2024-11-16T16:35:30,352 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:30,352 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:35:30,354 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-16T16:35:30,362 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,369 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:35:30,370 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:35:30,371 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:35:30,372 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:35:30,372 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:30,383 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:35:30,384 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:35:30,385 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store
2024-11-16T16:35:30,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:35:30,391 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:35:30,392 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:30,392 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:35:30,392 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:30,392 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:30,392 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:35:30,392 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:30,392 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:30,393 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774930392Disabling compacts and flushes for region at 1731774930392Disabling writes for close at 1731774930392Writing region close event to WAL at 1731774930392Closed at 1731774930392
2024-11-16T16:35:30,394 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/.initializing
2024-11-16T16:35:30,394 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/WALs/303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C35499%2C1731774929923, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/WALs/303fe01965e8,35499,1731774929923, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/oldWALs, maxLogs=10
2024-11-16T16:35:30,398 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35499%2C1731774929923.1731774930397
2024-11-16T16:35:30,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/WALs/303fe01965e8,35499,1731774929923/303fe01965e8%2C35499%2C1731774929923.1731774930397
2024-11-16T16:35:30,404 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35353:35353),(127.0.0.1/127.0.0.1:36287:36287)]
2024-11-16T16:35:30,405 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:35:30,405 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:30,405 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,405 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,407 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:30,408 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:35:30,408 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,409 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:30,409 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:30,411 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:35:30,411 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,411 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:30,412 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:30,414 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:35:30,414 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:30,415 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:30,416 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:35:30,416 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,417 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:30,417 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,418 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,419 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,421 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:35:30,422 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:30,424 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:35:30,425 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=776693, jitterRate=-0.012384563684463501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:35:30,426 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731774930405Initializing all the Stores at 1731774930406 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774930406Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774930406Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774930406Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774930406Cleaning up temporary data from old regions at 1731774930420 (+14 ms)Region opened successfully at 1731774930426 (+6 ms)
2024-11-16T16:35:30,426 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:35:30,430 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edec3b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:35:30,431 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:35:30,431 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:35:30,431 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:35:30,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:35:30,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:35:30,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:35:30,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:35:30,435 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:35:30,436 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:35:30,510 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:35:30,511 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:35:30,512 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:35:30,541 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:35:30,542 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:35:30,545 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:35:30,551 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:35:30,554 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:35:30,562 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:35:30,567 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:35:30,580 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:35:30,594 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:30,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:30,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,594 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,596 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,35499,1731774929923, sessionid=0x1014480579d0000, setting cluster-up flag (Was=false)
2024-11-16T16:35:30,614 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,647 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:35:30,651 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,674 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:30,710 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:35:30,714 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,35499,1731774929923
2024-11-16T16:35:30,717 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:35:30,721 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:30,722 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:35:30,722 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:35:30,722 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,35499,1731774929923
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:35:30,725 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:30,725 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:30,725 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:30,725 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:30,725 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:35:30,726 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,726 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:35:30,726 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,727 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731774960727
2024-11-16T16:35:30,727 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:35:30,727 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:35:30,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:35:30,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:35:30,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:35:30,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:35:30,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:35:30,729 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:30,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:35:30,729 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:35:30,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:35:30,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:35:30,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:35:30,730 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774930730,5,FailOnTimeoutGroup]
2024-11-16T16:35:30,730 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774930730,5,FailOnTimeoutGroup]
2024-11-16T16:35:30,730 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,730 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:35:30,730 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,731 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,731 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,731 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:35:30,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:35:30,740 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:35:30,741 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:35:30,742 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba
2024-11-16T16:35:30,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:35:30,751 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:35:30,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:30,754 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:35:30,756 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:35:30,756 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,757 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:30,757 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:35:30,759 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:35:30,759 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:30,760 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:35:30,762 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:35:30,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,762 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:30,763 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:35:30,764 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:35:30,764 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:30,765 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:30,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:35:30,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740
2024-11-16T16:35:30,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740
2024-11-16T16:35:30,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:35:30,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:35:30,768 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:35:30,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:35:30,772 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:35:30,772 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=735459, jitterRate=-0.06481662392616272}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:35:30,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731774930752Initializing all the Stores at 1731774930754 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774930754Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774930754Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774930754Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774930754Cleaning up temporary data from old regions at 1731774930767 (+13 ms)Region opened successfully at 1731774930773 (+6 ms)
2024-11-16T16:35:30,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:35:30,773 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:35:30,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:35:30,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:35:30,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:35:30,774 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:30,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774930773Disabling compacts and flushes for region at 1731774930773Disabling writes for close at 1731774930774 (+1 ms)Writing region close event to WAL at 1731774930774Closed at 1731774930774
2024-11-16T16:35:30,775 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:30,776 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:35:30,776 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(746): ClusterId : 982efa4d-21e9-479e-8f52-34456a664fc5
2024-11-16T16:35:30,776 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:35:30,776 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:35:30,777 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:35:30,778 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:35:30,784 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:35:30,784 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:35:30,794 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:35:30,795 DEBUG [RS:0;303fe01965e8:33199 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90a252d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:35:30,806 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:33199
2024-11-16T16:35:30,807 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:35:30,807 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:35:30,807 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:35:30,808 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,35499,1731774929923 with port=33199, startcode=1731774930246
2024-11-16T16:35:30,808 DEBUG [RS:0;303fe01965e8:33199 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:35:30,811 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36871, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:35:30,811 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35499 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,811 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35499 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,813 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba
2024-11-16T16:35:30,813 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43373
2024-11-16T16:35:30,813 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:35:30,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:35:30,826 DEBUG [RS:0;303fe01965e8:33199 {}] zookeeper.ZKUtil(111): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,826 WARN  [RS:0;303fe01965e8:33199 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:35:30,826 INFO  [RS:0;303fe01965e8:33199 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:30,826 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/WALs/303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,826 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,33199,1731774930246]
2024-11-16T16:35:30,830 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:35:30,833 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:35:30,833 INFO  [RS:0;303fe01965e8:33199 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:35:30,833 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,834 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:35:30,835 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:35:30,835 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,835 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,835 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,836 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,837 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,837 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:30,837 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:30,837 DEBUG [RS:0;303fe01965e8:33199 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:30,837 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,838 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,838 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,838 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,838 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,838 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,33199,1731774930246-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:35:30,851 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:35:30,852 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,33199,1731774930246-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,852 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,852 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.Replication(171): 303fe01965e8,33199,1731774930246 started
2024-11-16T16:35:30,865 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:30,865 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,33199,1731774930246, RpcServer on 303fe01965e8/172.17.0.2:33199, sessionid=0x1014480579d0001
2024-11-16T16:35:30,865 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:35:30,865 DEBUG [RS:0;303fe01965e8:33199 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,865 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,33199,1731774930246'
2024-11-16T16:35:30,865 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:35:30,866 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,33199,1731774930246
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,33199,1731774930246'
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:35:30,867 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:35:30,868 DEBUG [RS:0;303fe01965e8:33199 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:35:30,868 INFO  [RS:0;303fe01965e8:33199 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:35:30,868 INFO  [RS:0;303fe01965e8:33199 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:35:30,929 WARN  [303fe01965e8:35499 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:35:30,974 INFO  [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C33199%2C1731774930246, suffix=, logDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/WALs/303fe01965e8,33199,1731774930246, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/oldWALs, maxLogs=32
2024-11-16T16:35:30,980 INFO  [RS:0;303fe01965e8:33199 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33199%2C1731774930246.1731774930980
2024-11-16T16:35:30,987 INFO  [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/WALs/303fe01965e8,33199,1731774930246/303fe01965e8%2C33199%2C1731774930246.1731774930980
2024-11-16T16:35:30,991 DEBUG [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36287:36287),(127.0.0.1/127.0.0.1:35353:35353)]
2024-11-16T16:35:31,179 DEBUG [303fe01965e8:35499 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:35:31,181 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,33199,1731774930246
2024-11-16T16:35:31,185 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,33199,1731774930246, state=OPENING
2024-11-16T16:35:31,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:35:31,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:35:31,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-16T16:35:31,237 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:35:31,327 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:31,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:31,329 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:35:31,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:31,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:31,329 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,33199,1731774930246}]
2024-11-16T16:35:31,486 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:35:31,493 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56613, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:35:31,500 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:35:31,501 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:31,503 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C33199%2C1731774930246.meta, suffix=.meta, logDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/WALs/303fe01965e8,33199,1731774930246, archiveDir=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/oldWALs, maxLogs=32
2024-11-16T16:35:31,507 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33199%2C1731774930246.meta.1731774931506.meta
2024-11-16T16:35:31,513 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/WALs/303fe01965e8,33199,1731774930246/303fe01965e8%2C33199%2C1731774930246.meta.1731774931506.meta
2024-11-16T16:35:31,517 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36287:36287),(127.0.0.1/127.0.0.1:35353:35353)]
2024-11-16T16:35:31,521 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:35:31,521 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:35:31,521 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:35:31,521 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:35:31,521 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:35:31,521 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:31,522 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:35:31,522 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:35:31,523 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:35:31,524 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:35:31,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:31,525 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:31,525 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:35:31,526 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:35:31,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:31,527 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:31,527 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:35:31,528 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:35:31,528 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:31,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:31,529 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:35:31,530 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:35:31,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:31,531 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:31,532 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:35:31,533 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740
2024-11-16T16:35:31,535 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740
2024-11-16T16:35:31,537 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:35:31,537 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:35:31,538 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:35:31,540 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:35:31,541 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825038, jitterRate=0.04909096658229828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:35:31,542 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:35:31,542 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731774931522Writing region info on filesystem at 1731774931522Initializing all the Stores at 1731774931523 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774931523Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774931523Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774931523Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774931523Cleaning up temporary data from old regions at 1731774931537 (+14 ms)Running coprocessor post-open hooks at 1731774931542 (+5 ms)Region opened successfully at 1731774931542
2024-11-16T16:35:31,544 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731774931486
2024-11-16T16:35:31,546 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:35:31,547 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:35:31,548 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,33199,1731774930246
2024-11-16T16:35:31,549 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,33199,1731774930246, state=OPEN
2024-11-16T16:35:31,618 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:35:31,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:35:31,618 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,33199,1731774930246
2024-11-16T16:35:31,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:31,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:31,623 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:35:31,623 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,33199,1731774930246 in 289 msec
2024-11-16T16:35:31,629 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:35:31,629 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 847 msec
2024-11-16T16:35:31,630 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:31,630 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:35:31,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:35:31,633 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,33199,1731774930246, seqNum=-1]
2024-11-16T16:35:31,634 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:35:31,636 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40631, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:35:31,645 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 924 msec
2024-11-16T16:35:31,645 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731774931645, completionTime=-1
2024-11-16T16:35:31,645 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:35:31,645 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731774991648
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775051648
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,648 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,649 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:35499, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,649 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,649 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,652 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.335sec
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:35:31,656 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:35:31,659 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:35:31,659 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:35:31,659 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35499,1731774929923-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:31,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b23b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:31,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,35499,-1 for getting cluster id
2024-11-16T16:35:31,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:35:31,679 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '982efa4d-21e9-479e-8f52-34456a664fc5'
2024-11-16T16:35:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:35:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "982efa4d-21e9-479e-8f52-34456a664fc5"

2024-11-16T16:35:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e05bdfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,35499,-1]
2024-11-16T16:35:31,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:35:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,683 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:35:31,684 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78731c35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:31,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:35:31,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,33199,1731774930246, seqNum=-1]
2024-11-16T16:35:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:35:31,688 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:35:31,691 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,35499,1731774929923
2024-11-16T16:35:31,692 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:31,696 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:35:31,696 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:35:31,697 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:35:31,697 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:31,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,698 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:35:31,698 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:35:31,698 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1420279054, stopped=false
2024-11-16T16:35:31,698 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,35499,1731774929923
2024-11-16T16:35:31,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:31,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:31,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:31,720 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:31,720 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:31,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:31,720 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:35:31,720 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:35:31,721 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:31,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,721 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,33199,1731774930246' *****
2024-11-16T16:35:31,721 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:35:31,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:31,722 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:31,722 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:35:31,722 INFO  [RS:0;303fe01965e8:33199 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:35:31,722 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:35:31,722 INFO  [RS:0;303fe01965e8:33199 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,33199,1731774930246
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:33199.
2024-11-16T16:35:31,723 DEBUG [RS:0;303fe01965e8:33199 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:35:31,723 DEBUG [RS:0;303fe01965e8:33199 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:35:31,723 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:35:31,724 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-16T16:35:31,724 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-16T16:35:31,724 DEBUG [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T16:35:31,724 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:35:31,724 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:35:31,724 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:35:31,724 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:35:31,724 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:35:31,724 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-16T16:35:31,750 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/.tmp/ns/b4780bf797784a758d5d163f52d4a976 is 43, key is default/ns:d/1731774931637/Put/seqid=0
2024-11-16T16:35:31,756 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741835_1011 (size=5153)
2024-11-16T16:35:31,757 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741835_1011 (size=5153)
2024-11-16T16:35:31,757 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/.tmp/ns/b4780bf797784a758d5d163f52d4a976
2024-11-16T16:35:31,766 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/.tmp/ns/b4780bf797784a758d5d163f52d4a976 as hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/ns/b4780bf797784a758d5d163f52d4a976
2024-11-16T16:35:31,775 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/ns/b4780bf797784a758d5d163f52d4a976, entries=2, sequenceid=6, filesize=5.0 K
2024-11-16T16:35:31,777 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 53ms, sequenceid=6, compaction requested=false
2024-11-16T16:35:31,777 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:35:31,784 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-16T16:35:31,785 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:35:31,786 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:31,786 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774931724Running coprocessor pre-close hooks at 1731774931724Disabling compacts and flushes for region at 1731774931724Disabling writes for close at 1731774931724Obtaining lock to block concurrent updates at 1731774931724Preparing flush snapshotting stores in 1588230740 at 1731774931724Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731774931725 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731774931726 (+1 ms)Flushing 1588230740/ns: creating writer at 1731774931726Flushing 1588230740/ns: appending metadata at 1731774931749 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731774931749Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f5ce6c1: reopening flushed file at 1731774931765 (+16 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 53ms, sequenceid=6, compaction requested=false at 1731774931777 (+12 ms)Writing region close event to WAL at 1731774931779 (+2 ms)Running coprocessor post-close hooks at 1731774931785 (+6 ms)Closed at 1731774931786 (+1 ms)
2024-11-16T16:35:31,786 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:31,861 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T16:35:31,862 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T16:35:31,924 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,33199,1731774930246; all regions closed.
2024-11-16T16:35:31,925 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,925 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,925 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,925 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,925 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741834_1010 (size=1152)
2024-11-16T16:35:31,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741834_1010 (size=1152)
2024-11-16T16:35:31,931 DEBUG [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/oldWALs
2024-11-16T16:35:31,931 INFO  [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C33199%2C1731774930246.meta:.meta(num 1731774931506)
2024-11-16T16:35:31,932 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,932 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,932 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,932 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,932 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:31,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741833_1009 (size=93)
2024-11-16T16:35:31,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741833_1009 (size=93)
2024-11-16T16:35:31,937 DEBUG [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/oldWALs
2024-11-16T16:35:31,937 INFO  [RS:0;303fe01965e8:33199 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C33199%2C1731774930246:(num 1731774930980)
2024-11-16T16:35:31,937 DEBUG [RS:0;303fe01965e8:33199 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:31,937 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:35:31,938 INFO  [RS:0;303fe01965e8:33199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:35:31,938 INFO  [RS:0;303fe01965e8:33199 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T16:35:31,938 INFO  [RS:0;303fe01965e8:33199 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:35:31,938 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:35:31,938 INFO  [RS:0;303fe01965e8:33199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33199
2024-11-16T16:35:31,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:35:31,951 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,33199,1731774930246
2024-11-16T16:35:31,951 INFO  [RS:0;303fe01965e8:33199 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:35:31,962 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,33199,1731774930246]
2024-11-16T16:35:31,972 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,33199,1731774930246 already deleted, retry=false
2024-11-16T16:35:31,972 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,33199,1731774930246 expired; onlineServers=0
2024-11-16T16:35:31,973 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,35499,1731774929923' *****
2024-11-16T16:35:31,973 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:35:31,973 INFO  [M:0;303fe01965e8:35499 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:35:31,973 INFO  [M:0;303fe01965e8:35499 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:35:31,973 DEBUG [M:0;303fe01965e8:35499 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:35:31,973 DEBUG [M:0;303fe01965e8:35499 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:35:31,973 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:35:31,973 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774930730 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774930730,5,FailOnTimeoutGroup]
2024-11-16T16:35:31,973 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774930730 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774930730,5,FailOnTimeoutGroup]
2024-11-16T16:35:31,974 INFO  [M:0;303fe01965e8:35499 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:35:31,974 INFO  [M:0;303fe01965e8:35499 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:35:31,974 DEBUG [M:0;303fe01965e8:35499 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:35:31,974 INFO  [M:0;303fe01965e8:35499 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:35:31,974 INFO  [M:0;303fe01965e8:35499 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:35:31,974 INFO  [M:0;303fe01965e8:35499 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:35:31,974 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:35:31,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:35:31,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:31,983 DEBUG [M:0;303fe01965e8:35499 {}] zookeeper.ZKUtil(347): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:35:31,984 WARN  [M:0;303fe01965e8:35499 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:35:31,984 INFO  [M:0;303fe01965e8:35499 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/.lastflushedseqids
2024-11-16T16:35:31,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741836_1012 (size=99)
2024-11-16T16:35:31,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741836_1012 (size=99)
2024-11-16T16:35:31,995 INFO  [M:0;303fe01965e8:35499 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:35:31,995 INFO  [M:0;303fe01965e8:35499 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:35:31,995 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:35:31,995 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:31,995 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:31,995 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:35:31,995 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:31,996 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-16T16:35:32,013 DEBUG [M:0;303fe01965e8:35499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff2c708df26f4d3fafbc261765442786 is 82, key is hbase:meta,,1/info:regioninfo/1731774931547/Put/seqid=0
2024-11-16T16:35:32,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741837_1013 (size=5672)
2024-11-16T16:35:32,020 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741837_1013 (size=5672)
2024-11-16T16:35:32,021 INFO  [M:0;303fe01965e8:35499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff2c708df26f4d3fafbc261765442786
2024-11-16T16:35:32,042 DEBUG [M:0;303fe01965e8:35499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f20a20994f49c7b82850a8c26b07c6 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731774931643/Put/seqid=0
2024-11-16T16:35:32,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741838_1014 (size=5275)
2024-11-16T16:35:32,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741838_1014 (size=5275)
2024-11-16T16:35:32,048 INFO  [M:0;303fe01965e8:35499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f20a20994f49c7b82850a8c26b07c6
2024-11-16T16:35:32,062 INFO  [RS:0;303fe01965e8:33199 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:35:32,062 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:32,062 DEBUG [pool-180-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33199-0x1014480579d0001, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:32,062 INFO  [RS:0;303fe01965e8:33199 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,33199,1731774930246; zookeeper connection closed.
2024-11-16T16:35:32,062 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ae8c2af {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ae8c2af
2024-11-16T16:35:32,063 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:35:32,069 DEBUG [M:0;303fe01965e8:35499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cb86814c40ca4e22b5a154dbdcbbd168 is 69, key is 303fe01965e8,33199,1731774930246/rs:state/1731774930812/Put/seqid=0
2024-11-16T16:35:32,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741839_1015 (size=5156)
2024-11-16T16:35:32,074 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741839_1015 (size=5156)
2024-11-16T16:35:32,075 INFO  [M:0;303fe01965e8:35499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cb86814c40ca4e22b5a154dbdcbbd168
2024-11-16T16:35:32,096 DEBUG [M:0;303fe01965e8:35499 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b65a97868f09410fba05a6db0f9f2c8f is 52, key is load_balancer_on/state:d/1731774931695/Put/seqid=0
2024-11-16T16:35:32,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741840_1016 (size=5056)
2024-11-16T16:35:32,101 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741840_1016 (size=5056)
2024-11-16T16:35:32,101 INFO  [M:0;303fe01965e8:35499 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b65a97868f09410fba05a6db0f9f2c8f
2024-11-16T16:35:32,108 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ff2c708df26f4d3fafbc261765442786 as hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ff2c708df26f4d3fafbc261765442786
2024-11-16T16:35:32,114 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ff2c708df26f4d3fafbc261765442786, entries=8, sequenceid=29, filesize=5.5 K
2024-11-16T16:35:32,115 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/08f20a20994f49c7b82850a8c26b07c6 as hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08f20a20994f49c7b82850a8c26b07c6
2024-11-16T16:35:32,123 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/08f20a20994f49c7b82850a8c26b07c6, entries=3, sequenceid=29, filesize=5.2 K
2024-11-16T16:35:32,125 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cb86814c40ca4e22b5a154dbdcbbd168 as hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cb86814c40ca4e22b5a154dbdcbbd168
2024-11-16T16:35:32,132 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cb86814c40ca4e22b5a154dbdcbbd168, entries=1, sequenceid=29, filesize=5.0 K
2024-11-16T16:35:32,134 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b65a97868f09410fba05a6db0f9f2c8f as hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b65a97868f09410fba05a6db0f9f2c8f
2024-11-16T16:35:32,141 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43373/user/jenkins/test-data/a1a12371-e93e-8754-e570-2f3a1fb912ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b65a97868f09410fba05a6db0f9f2c8f, entries=1, sequenceid=29, filesize=4.9 K
2024-11-16T16:35:32,143 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=29, compaction requested=false
2024-11-16T16:35:32,144 INFO  [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:32,144 DEBUG [M:0;303fe01965e8:35499 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774931995Disabling compacts and flushes for region at 1731774931995Disabling writes for close at 1731774931995Obtaining lock to block concurrent updates at 1731774931996 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731774931996Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731774931996Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731774931997 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731774931997Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731774932013 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731774932013Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731774932027 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731774932042 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731774932042Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731774932054 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731774932068 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731774932069 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731774932080 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731774932095 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731774932095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36264b1f: reopening flushed file at 1731774932107 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42ed9f8b: reopening flushed file at 1731774932114 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b22e0b8: reopening flushed file at 1731774932123 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e1c946: reopening flushed file at 1731774932132 (+9 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 147ms, sequenceid=29, compaction requested=false at 1731774932143 (+11 ms)Writing region close event to WAL at 1731774932144 (+1 ms)Closed at 1731774932144
2024-11-16T16:35:32,145 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:32,145 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:32,145 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:32,145 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:32,145 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:32,147 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741830_1006 (size=10311)
2024-11-16T16:35:32,148 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44815 is added to blk_1073741830_1006 (size=10311)
2024-11-16T16:35:32,148 INFO  [M:0;303fe01965e8:35499 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:35:32,148 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:35:32,148 INFO  [M:0;303fe01965e8:35499 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35499
2024-11-16T16:35:32,149 INFO  [M:0;303fe01965e8:35499 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:35:32,236 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:35:32,242 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:32,259 INFO  [M:0;303fe01965e8:35499 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:35:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35499-0x1014480579d0000, quorum=127.0.0.1:57432, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:35:32,268 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a329c51{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:32,269 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d8729f1{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:32,269 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:32,269 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a17cc25{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:32,269 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:32,269 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7130e643{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:32,283 WARN  [BP-353393911-172.17.0.2-1731774927787 heartbeating to localhost/127.0.0.1:43373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:32,283 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:32,283 WARN  [BP-353393911-172.17.0.2-1731774927787 heartbeating to localhost/127.0.0.1:43373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-353393911-172.17.0.2-1731774927787 (Datanode Uuid 15741f7f-3d4e-4e3d-a023-c72960fb091b) service to localhost/127.0.0.1:43373
2024-11-16T16:35:32,283 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:32,284 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data3/current/BP-353393911-172.17.0.2-1731774927787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:32,284 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data4/current/BP-353393911-172.17.0.2-1731774927787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:32,284 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:32,293 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@77ff58b7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:32,294 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9d8d6d6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:32,294 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:32,294 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39fb5954{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:32,294 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5357e169{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:32,296 WARN  [BP-353393911-172.17.0.2-1731774927787 heartbeating to localhost/127.0.0.1:43373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:32,296 WARN  [BP-353393911-172.17.0.2-1731774927787 heartbeating to localhost/127.0.0.1:43373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-353393911-172.17.0.2-1731774927787 (Datanode Uuid 0c31588d-7536-490d-afab-1ab1bcaad24d) service to localhost/127.0.0.1:43373
2024-11-16T16:35:32,296 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data1/current/BP-353393911-172.17.0.2-1731774927787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:32,297 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/cluster_8adea76c-5d55-6614-394d-5b45f0170c9b/data/data2/current/BP-353393911-172.17.0.2-1731774927787 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:32,297 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:32,297 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:32,297 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:32,305 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bc93f4{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:35:32,306 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ec80534{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:32,306 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:32,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69fa0c88{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:32,306 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5686c999{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:32,314 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:35:32,337 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.log.dir so I do NOT create it in target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d8cc6c2e-cc4c-451f-1bb0-8ce7a2d898ab/hadoop.tmp.dir so I do NOT create it in target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c, deleteOnExit=true
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:35:32,338 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/test.cache.data in system properties and HBase conf
2024-11-16T16:35:32,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:35:32,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:35:32,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:35:32,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:35:32,339 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:35:32,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:35:32,340 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:35:32,341 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:35:32,360 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:35:32,689 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:32,694 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:32,696 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:32,696 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:32,696 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:35:32,697 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:32,697 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@57980111{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:32,698 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48202e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:32,793 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2cd29190{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-32853-hadoop-hdfs-3_4_1-tests_jar-_-any-7590663587574965954/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:35:32,793 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3cc2b5d2{HTTP/1.1, (http/1.1)}{localhost:32853}
2024-11-16T16:35:32,794 INFO  [Time-limited test {}] server.Server(415): Started @108569ms
2024-11-16T16:35:32,806 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:35:32,838 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:35:33,032 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:33,036 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:33,037 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:33,037 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:33,037 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:35:33,038 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@357bf795{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:33,038 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@149f32b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:33,138 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1705bda4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-43515-hadoop-hdfs-3_4_1-tests_jar-_-any-2371820488086304015/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:33,138 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2cbd3715{HTTP/1.1, (http/1.1)}{localhost:43515}
2024-11-16T16:35:33,139 INFO  [Time-limited test {}] server.Server(415): Started @108914ms
2024-11-16T16:35:33,140 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:33,166 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:33,170 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:33,170 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:33,170 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:33,170 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:35:33,172 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f400306{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:33,173 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@630b1d87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:33,284 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19a6c3ca{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-33305-hadoop-hdfs-3_4_1-tests_jar-_-any-1850039742459493044/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:33,284 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e259212{HTTP/1.1, (http/1.1)}{localhost:33305}
2024-11-16T16:35:33,284 INFO  [Time-limited test {}] server.Server(415): Started @109060ms
2024-11-16T16:35:33,286 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:34,116 WARN  [Thread-672 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data1/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:34,117 WARN  [Thread-673 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data2/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:34,142 WARN  [Thread-637 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:34,145 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6d42c145244b2c2 with lease ID 0xa651c37709922912: Processing first storage report for DS-73e39b61-3448-4c69-af4d-56105bf53d27 from datanode DatanodeRegistration(127.0.0.1:41585, datanodeUuid=78d4b138-0b42-4053-b5f7-f4f9e469c507, infoPort=36413, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:34,145 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6d42c145244b2c2 with lease ID 0xa651c37709922912: from storage DS-73e39b61-3448-4c69-af4d-56105bf53d27 node DatanodeRegistration(127.0.0.1:41585, datanodeUuid=78d4b138-0b42-4053-b5f7-f4f9e469c507, infoPort=36413, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:34,145 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc6d42c145244b2c2 with lease ID 0xa651c37709922912: Processing first storage report for DS-6cc382bf-fbab-4ab7-a41b-64452f4b7747 from datanode DatanodeRegistration(127.0.0.1:41585, datanodeUuid=78d4b138-0b42-4053-b5f7-f4f9e469c507, infoPort=36413, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:34,145 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc6d42c145244b2c2 with lease ID 0xa651c37709922912: from storage DS-6cc382bf-fbab-4ab7-a41b-64452f4b7747 node DatanodeRegistration(127.0.0.1:41585, datanodeUuid=78d4b138-0b42-4053-b5f7-f4f9e469c507, infoPort=36413, infoSecurePort=0, ipcPort=46543, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:34,365 WARN  [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data3/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:34,366 WARN  [Thread-685 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data4/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:34,387 WARN  [Thread-660 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:34,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdaa4dc1d009a1a07 with lease ID 0xa651c37709922913: Processing first storage report for DS-7da5a248-5b7c-4cd4-840d-4b120725ae67 from datanode DatanodeRegistration(127.0.0.1:42837, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=46787, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:34,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdaa4dc1d009a1a07 with lease ID 0xa651c37709922913: from storage DS-7da5a248-5b7c-4cd4-840d-4b120725ae67 node DatanodeRegistration(127.0.0.1:42837, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=46787, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:35:34,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdaa4dc1d009a1a07 with lease ID 0xa651c37709922913: Processing first storage report for DS-5460f134-4094-4b7c-8c9d-3a245d22089a from datanode DatanodeRegistration(127.0.0.1:42837, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=46787, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:34,390 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdaa4dc1d009a1a07 with lease ID 0xa651c37709922913: from storage DS-5460f134-4094-4b7c-8c9d-3a245d22089a node DatanodeRegistration(127.0.0.1:42837, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=46787, infoSecurePort=0, ipcPort=35479, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:34,435 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a
2024-11-16T16:35:34,438 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/zookeeper_0, clientPort=54861, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:35:34,439 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54861
2024-11-16T16:35:34,440 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,441 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:35:34,454 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:35:34,455 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4 with version=8
2024-11-16T16:35:34,456 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:35:34,458 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:35:34,458 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:35:34,459 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:35:34,460 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39189
2024-11-16T16:35:34,462 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39189 connecting to ZooKeeper ensemble=127.0.0.1:54861
2024-11-16T16:35:34,520 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391890x0, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:35:34,521 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39189-0x101448069510000 connected
2024-11-16T16:35:34,604 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,607 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,610 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:34,611 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4, hbase.cluster.distributed=false
2024-11-16T16:35:34,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:35:34,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39189
2024-11-16T16:35:34,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39189
2024-11-16T16:35:34,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39189
2024-11-16T16:35:34,614 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39189
2024-11-16T16:35:34,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39189
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:35:34,632 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:35:34,633 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36407
2024-11-16T16:35:34,634 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36407 connecting to ZooKeeper ensemble=127.0.0.1:54861
2024-11-16T16:35:34,635 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,637 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364070x0, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:35:34,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:364070x0, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:35:34,646 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36407-0x101448069510001 connected
2024-11-16T16:35:34,647 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:35:34,647 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:35:34,648 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:35:34,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:35:34,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36407
2024-11-16T16:35:34,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36407
2024-11-16T16:35:34,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36407
2024-11-16T16:35:34,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36407
2024-11-16T16:35:34,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36407
2024-11-16T16:35:34,663 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:39189
2024-11-16T16:35:34,664 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:34,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:34,675 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:35:34,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,685 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:35:34,686 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,39189,1731774934458 from backup master directory
2024-11-16T16:35:34,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:34,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:35:34,698 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:35:34,699 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,707 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/hbase.id] with ID: d9e45549-8bbb-4afb-ad11-c315a134fc74
2024-11-16T16:35:34,707 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/.tmp/hbase.id
2024-11-16T16:35:34,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:35:34,714 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:35:34,714 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/.tmp/hbase.id]:[hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/hbase.id]
2024-11-16T16:35:34,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:34,729 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:35:34,731 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-16T16:35:34,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:35:34,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:35:34,750 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:35:34,751 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:35:34,752 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:34,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:35:34,760 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:35:34,761 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store
2024-11-16T16:35:34,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:35:34,767 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:35:34,768 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:34,769 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:35:34,769 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:34,769 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:34,769 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:35:34,769 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:34,769 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:35:34,769 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774934769Disabling compacts and flushes for region at 1731774934769Disabling writes for close at 1731774934769Writing region close event to WAL at 1731774934769Closed at 1731774934769
2024-11-16T16:35:34,770 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/.initializing
2024-11-16T16:35:34,770 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C39189%2C1731774934458, suffix=, logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458, archiveDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/oldWALs, maxLogs=10
2024-11-16T16:35:34,774 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C39189%2C1731774934458.1731774934774
2024-11-16T16:35:34,779 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774
2024-11-16T16:35:34,780 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36413:36413),(127.0.0.1/127.0.0.1:46787:46787)]
2024-11-16T16:35:34,782 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:35:34,782 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:34,783 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,783 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,784 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:34,786 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:35:34,786 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:34,786 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:34,786 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:34,788 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:35:34,788 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:34,788 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:34,788 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:34,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:35:34,790 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:34,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:34,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:35:34,791 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:35:34,791 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:34,792 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:34,792 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,793 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,793 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:35:34,797 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:35:34,799 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:35:34,799 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715940, jitterRate=-0.0896354615688324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:35:34,800 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731774934783Initializing all the Stores at 1731774934784 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774934784Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774934784Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774934784Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774934784Cleaning up temporary data from old regions at 1731774934795 (+11 ms)Region opened successfully at 1731774934800 (+5 ms)
2024-11-16T16:35:34,800 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:35:34,804 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af491fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:35:34,805 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:35:34,805 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:35:34,805 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:35:34,805 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:35:34,806 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:35:34,806 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:35:34,806 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:35:34,809 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:35:34,810 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:35:34,821 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:35:34,822 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:35:34,823 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:35:34,835 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:35:34,836 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:35:34,837 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:35:34,846 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:35:34,847 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:35:34,856 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:35:34,859 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:35:34,867 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:35:34,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:34,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:35:34,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,878 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,39189,1731774934458, sessionid=0x101448069510000, setting cluster-up flag (Was=false)
2024-11-16T16:35:34,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,930 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:35:34,931 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:34,983 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:35:34,984 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39189,1731774934458
2024-11-16T16:35:34,985 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:35:34,987 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:34,988 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:35:34,988 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:35:34,988 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,39189,1731774934458
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:35:34,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731774964992
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:35:34,992 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:34,992 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:34,992 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:35:34,993 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:35:34,993 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:35:34,993 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:35:34,993 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:35:34,993 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:35:34,993 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774934993,5,FailOnTimeoutGroup]
2024-11-16T16:35:34,994 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774934993,5,FailOnTimeoutGroup]
2024-11-16T16:35:34,994 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:34,994 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:35:34,994 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:34,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:34,994 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:34,994 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:35:35,000 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:35:35,001 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:35:35,002 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:35:35,002 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4
2024-11-16T16:35:35,009 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:35:35,009 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:35:35,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:35,012 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:35:35,013 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:35:35,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,014 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,014 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:35:35,015 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:35:35,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,016 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,016 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:35:35,017 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:35:35,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,018 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,018 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:35:35,019 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:35:35,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,020 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:35:35,021 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740
2024-11-16T16:35:35,021 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740
2024-11-16T16:35:35,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:35:35,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:35:35,023 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:35:35,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:35:35,026 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:35:35,027 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784840, jitterRate=-0.0020250678062438965}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731774935010Initializing all the Stores at 1731774935011 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935011Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935011Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774935011Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935012 (+1 ms)Cleaning up temporary data from old regions at 1731774935023 (+11 ms)Region opened successfully at 1731774935028 (+5 ms)
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:35:35,028 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:35:35,028 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:35:35,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774935028Disabling compacts and flushes for region at 1731774935028Disabling writes for close at 1731774935028Writing region close event to WAL at 1731774935028Closed at 1731774935028
2024-11-16T16:35:35,030 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:35,030 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:35:35,030 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:35:35,031 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:35:35,032 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:35:35,052 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(746): ClusterId : d9e45549-8bbb-4afb-ad11-c315a134fc74
2024-11-16T16:35:35,052 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:35:35,064 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:35:35,064 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:35:35,075 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:35:35,075 DEBUG [RS:0;303fe01965e8:36407 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7612cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:35:35,089 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:36407
2024-11-16T16:35:35,089 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:35:35,089 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:35:35,089 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:35:35,090 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,39189,1731774934458 with port=36407, startcode=1731774934631
2024-11-16T16:35:35,090 DEBUG [RS:0;303fe01965e8:36407 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:35:35,092 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49527, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:35:35,093 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,093 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,095 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4
2024-11-16T16:35:35,095 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36911
2024-11-16T16:35:35,095 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:35:35,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:35:35,106 DEBUG [RS:0;303fe01965e8:36407 {}] zookeeper.ZKUtil(111): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,106 WARN  [RS:0;303fe01965e8:36407 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:35:35,106 INFO  [RS:0;303fe01965e8:36407 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:35,107 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,107 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,36407,1731774934631]
2024-11-16T16:35:35,110 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:35:35,114 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:35:35,114 INFO  [RS:0;303fe01965e8:36407 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:35:35,114 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,114 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:35:35,115 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:35:35,115 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,115 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:35,116 DEBUG [RS:0;303fe01965e8:36407 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,118 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,36407,1731774934631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:35:35,155 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:35:35,155 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,36407,1731774934631-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,156 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,156 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.Replication(171): 303fe01965e8,36407,1731774934631 started
2024-11-16T16:35:35,168 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,168 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,36407,1731774934631, RpcServer on 303fe01965e8/172.17.0.2:36407, sessionid=0x101448069510001
2024-11-16T16:35:35,168 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:35:35,168 DEBUG [RS:0;303fe01965e8:36407 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,168 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,36407,1731774934631'
2024-11-16T16:35:35,168 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:35:35,169 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,36407,1731774934631'
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:35:35,170 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:35:35,171 DEBUG [RS:0;303fe01965e8:36407 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:35:35,171 INFO  [RS:0;303fe01965e8:36407 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:35:35,171 INFO  [RS:0;303fe01965e8:36407 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:35:35,183 WARN  [303fe01965e8:39189 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:35:35,276 INFO  [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C36407%2C1731774934631, suffix=, logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631, archiveDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs, maxLogs=32
2024-11-16T16:35:35,277 INFO  [RS:0;303fe01965e8:36407 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36407%2C1731774934631.1731774935277
2024-11-16T16:35:35,287 INFO  [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
2024-11-16T16:35:35,289 DEBUG [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46787:46787),(127.0.0.1/127.0.0.1:36413:36413)]
2024-11-16T16:35:35,433 DEBUG [303fe01965e8:39189 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:35:35,434 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,436 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,36407,1731774934631, state=OPENING
2024-11-16T16:35:35,446 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:35:35,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:35,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:35:35,459 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:35:35,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:35,459 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:35,459 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,36407,1731774934631}]
2024-11-16T16:35:35,617 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:35:35,622 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:35:35,628 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:35:35,629 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:35,631 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C36407%2C1731774934631.meta, suffix=.meta, logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631, archiveDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs, maxLogs=32
2024-11-16T16:35:35,631 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
2024-11-16T16:35:35,636 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
2024-11-16T16:35:35,638 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36413:36413),(127.0.0.1/127.0.0.1:46787:46787)]
2024-11-16T16:35:35,638 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:35:35,639 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:35:35,639 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:35:35,641 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:35:35,642 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:35:35,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,643 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:35:35,644 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:35:35,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,645 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,645 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:35:35,646 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:35:35,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,647 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,647 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:35:35,648 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:35:35,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,649 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:35:35,649 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:35:35,649 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740
2024-11-16T16:35:35,650 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740
2024-11-16T16:35:35,652 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:35:35,652 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:35:35,652 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:35:35,654 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:35:35,654 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=806108, jitterRate=0.025019347667694092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:35:35,655 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:35:35,655 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731774935640Writing region info on filesystem at 1731774935640Initializing all the Stores at 1731774935641 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935641Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935641Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774935641Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774935641Cleaning up temporary data from old regions at 1731774935652 (+11 ms)Running coprocessor post-open hooks at 1731774935655 (+3 ms)Region opened successfully at 1731774935655
2024-11-16T16:35:35,656 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731774935616
2024-11-16T16:35:35,659 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:35:35,659 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:35:35,660 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,661 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,36407,1731774934631, state=OPEN
2024-11-16T16:35:35,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:35:35,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:35:35,716 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,36407,1731774934631
2024-11-16T16:35:35,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:35,716 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:35:35,719 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:35:35,719 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,36407,1731774934631 in 257 msec
2024-11-16T16:35:35,722 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:35:35,722 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 689 msec
2024-11-16T16:35:35,723 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:35:35,723 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:35:35,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:35:35,725 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,36407,1731774934631, seqNum=-1]
2024-11-16T16:35:35,725 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:35:35,726 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56851, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:35:35,733 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 745 msec
2024-11-16T16:35:35,733 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731774935733, completionTime=-1
2024-11-16T16:35:35,733 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:35:35,733 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:35:35,735 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:35:35,735 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731774995735
2024-11-16T16:35:35,735 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775055735
2024-11-16T16:35:35,735 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:39189, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,736 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,738 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:35:35,740 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.041sec
2024-11-16T16:35:35,740 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:35:35,740 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:35:35,740 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:35:35,740 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:35:35,741 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:35:35,741 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:35:35,741 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:35:35,743 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:35:35,743 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:35:35,744 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39189,1731774934458-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,753 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a6b9e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:35,753 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,39189,-1 for getting cluster id
2024-11-16T16:35:35,753 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:35:35,755 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9e45549-8bbb-4afb-ad11-c315a134fc74'
2024-11-16T16:35:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:35:35,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9e45549-8bbb-4afb-ad11-c315a134fc74"

2024-11-16T16:35:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dafd83e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,39189,-1]
2024-11-16T16:35:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:35:35,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:35:35,758 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:35:35,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@664ad8e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:35:35,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:35:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,36407,1731774934631, seqNum=-1]
2024-11-16T16:35:35,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:35:35,762 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:35:35,764 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,39189,1731774934458
2024-11-16T16:35:35,764 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:35,766 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:35:35,781 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:35:35,782 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33539
2024-11-16T16:35:35,783 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33539 connecting to ZooKeeper ensemble=127.0.0.1:54861
2024-11-16T16:35:35,784 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:35,785 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:35:35,804 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335390x0, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:35:35,804 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-16T16:35:35,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33539-0x101448069510002 connected
2024-11-16T16:35:35,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-16T16:35:35,805 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:35:35,807 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:35:35,807 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:35:35,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:35:35,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33539
2024-11-16T16:35:35,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33539
2024-11-16T16:35:35,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33539
2024-11-16T16:35:35,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33539
2024-11-16T16:35:35,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33539
2024-11-16T16:35:35,816 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(746): ClusterId : d9e45549-8bbb-4afb-ad11-c315a134fc74
2024-11-16T16:35:35,816 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:35:35,826 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:35:35,826 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:35:35,836 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:35:35,837 DEBUG [RS:1;303fe01965e8:33539 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de4b550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:35:35,850 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;303fe01965e8:33539
2024-11-16T16:35:35,850 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:35:35,850 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:35:35,850 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:35:35,851 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,39189,1731774934458 with port=33539, startcode=1731774935781
2024-11-16T16:35:35,851 DEBUG [RS:1;303fe01965e8:33539 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:35:35,853 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59605, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:35:35,853 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,853 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39189 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,855 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4
2024-11-16T16:35:35,855 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36911
2024-11-16T16:35:35,855 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:35:35,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:35:35,867 DEBUG [RS:1;303fe01965e8:33539 {}] zookeeper.ZKUtil(111): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,867 WARN  [RS:1;303fe01965e8:33539 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:35:35,868 INFO  [RS:1;303fe01965e8:33539 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:35:35,868 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,33539,1731774935781]
2024-11-16T16:35:35,868 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,873 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:35:35,876 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:35:35,877 INFO  [RS:1;303fe01965e8:33539 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:35:35,877 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,878 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:35:35,879 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:35:35,879 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,879 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,879 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:35,880 DEBUG [RS:1;303fe01965e8:33539 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:35:35,883 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,884 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,884 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,884 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,884 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,884 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,33539,1731774935781-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:35:35,900 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:35:35,900 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,33539,1731774935781-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,901 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,901 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.Replication(171): 303fe01965e8,33539,1731774935781 started
2024-11-16T16:35:35,914 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:35:35,914 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,33539,1731774935781, RpcServer on 303fe01965e8/172.17.0.2:33539, sessionid=0x101448069510002
2024-11-16T16:35:35,914 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:35:35,914 DEBUG [RS:1;303fe01965e8:33539 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,914 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;303fe01965e8:33539,5,FailOnTimeoutGroup]
2024-11-16T16:35:35,914 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,33539,1731774935781'
2024-11-16T16:35:35,914 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:35:35,914 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:35:35,915 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,33539,1731774935781
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,33539,1731774935781'
2024-11-16T16:35:35,915 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:35:35,916 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:35:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,39189,1731774934458
2024-11-16T16:35:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@53319cf8
2024-11-16T16:35:35,916 DEBUG [RS:1;303fe01965e8:33539 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:35:35,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T16:35:35,916 INFO  [RS:1;303fe01965e8:33539 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:35:35,916 INFO  [RS:1;303fe01965e8:33539 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:35:35,918 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T16:35:35,918 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-16T16:35:35,919 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-16T16:35:35,919 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:35:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-16T16:35:35,922 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T16:35:35,922 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:35,922 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-16T16:35:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:35:35,923 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T16:35:35,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741835_1011 (size=393)
2024-11-16T16:35:35,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741835_1011 (size=393)
2024-11-16T16:35:36,019 INFO  [RS:1;303fe01965e8:33539 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C33539%2C1731774935781, suffix=, logDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781, archiveDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs, maxLogs=32
2024-11-16T16:35:36,020 INFO  [RS:1;303fe01965e8:33539 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774936019
2024-11-16T16:35:36,027 INFO  [RS:1;303fe01965e8:33539 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019
2024-11-16T16:35:36,028 DEBUG [RS:1;303fe01965e8:33539 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36413:36413),(127.0.0.1/127.0.0.1:46787:46787)]
2024-11-16T16:35:36,339 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3eddc71f8121fc89cc96f04482c1d1c1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4
2024-11-16T16:35:36,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41585 is added to blk_1073741837_1013 (size=76)
2024-11-16T16:35:36,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42837 is added to blk_1073741837_1013 (size=76)
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 3eddc71f8121fc89cc96f04482c1d1c1, disabling compactions & flushes
2024-11-16T16:35:36,352 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1. after waiting 0 ms
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,352 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,352 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3eddc71f8121fc89cc96f04482c1d1c1:
Waiting for close lock at 1731774936352Disabling compacts and flushes for region at 1731774936352Disabling writes for close at 1731774936352Writing region close event to WAL at 1731774936352Closed at 1731774936352
2024-11-16T16:35:36,354 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T16:35:36,355 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731774936354"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731774936354"}]},"ts":"1731774936354"}
2024-11-16T16:35:36,358 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T16:35:36,360 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T16:35:36,360 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774936360"}]},"ts":"1731774936360"}
2024-11-16T16:35:36,363 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-16T16:35:36,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {303fe01965e8=0} racks are {/default-rack=0}
2024-11-16T16:35:36,371 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-11-16T16:35:36,371 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-11-16T16:35:36,371 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-11-16T16:35:36,371 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-11-16T16:35:36,371 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-11-16T16:35:36,372 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-11-16T16:35:36,372 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-11-16T16:35:36,373 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3eddc71f8121fc89cc96f04482c1d1c1, ASSIGN}]
2024-11-16T16:35:36,375 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3eddc71f8121fc89cc96f04482c1d1c1, ASSIGN
2024-11-16T16:35:36,376 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3eddc71f8121fc89cc96f04482c1d1c1, ASSIGN; state=OFFLINE, location=303fe01965e8,33539,1731774935781; forceNewPlan=false, retain=false
2024-11-16T16:35:36,527 INFO  [303fe01965e8:39189 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-16T16:35:36,528 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3eddc71f8121fc89cc96f04482c1d1c1, regionState=OPENING, regionLocation=303fe01965e8,33539,1731774935781
2024-11-16T16:35:36,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3eddc71f8121fc89cc96f04482c1d1c1, ASSIGN because future has completed
2024-11-16T16:35:36,533 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eddc71f8121fc89cc96f04482c1d1c1, server=303fe01965e8,33539,1731774935781}]
2024-11-16T16:35:36,690 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:35:36,695 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47577, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:35:36,700 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,700 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3eddc71f8121fc89cc96f04482c1d1c1, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:35:36,701 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,701 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:35:36,701 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,701 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,703 INFO  [StoreOpener-3eddc71f8121fc89cc96f04482c1d1c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 3eddc71f8121fc89cc96f04482c1d1c1 
2024-11-16T16:35:36,705 INFO  [StoreOpener-3eddc71f8121fc89cc96f04482c1d1c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3eddc71f8121fc89cc96f04482c1d1c1 columnFamilyName info
2024-11-16T16:35:36,705 DEBUG [StoreOpener-3eddc71f8121fc89cc96f04482c1d1c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:35:36,706 INFO  [StoreOpener-3eddc71f8121fc89cc96f04482c1d1c1-1 {}] regionserver.HStore(327): Store=3eddc71f8121fc89cc96f04482c1d1c1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:35:36,706 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,707 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,708 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,709 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,709 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,712 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,715 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:35:36,716 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3eddc71f8121fc89cc96f04482c1d1c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=793731, jitterRate=0.009281858801841736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:35:36,716 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:36,717 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3eddc71f8121fc89cc96f04482c1d1c1:
Running coprocessor pre-open hook at 1731774936702Writing region info on filesystem at 1731774936702Initializing all the Stores at 1731774936703 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774936703Cleaning up temporary data from old regions at 1731774936709 (+6 ms)Running coprocessor post-open hooks at 1731774936716 (+7 ms)Region opened successfully at 1731774936717 (+1 ms)
2024-11-16T16:35:36,719 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., pid=6, masterSystemTime=1731774936689
2024-11-16T16:35:36,722 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,722 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:36,723 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3eddc71f8121fc89cc96f04482c1d1c1, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,33539,1731774935781
2024-11-16T16:35:36,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3eddc71f8121fc89cc96f04482c1d1c1, server=303fe01965e8,33539,1731774935781 because future has completed
2024-11-16T16:35:36,732 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T16:35:36,733 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3eddc71f8121fc89cc96f04482c1d1c1, server=303fe01965e8,33539,1731774935781 in 195 msec
2024-11-16T16:35:36,735 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T16:35:36,735 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=3eddc71f8121fc89cc96f04482c1d1c1, ASSIGN in 359 msec
2024-11-16T16:35:36,736 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T16:35:36,736 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774936736"}]},"ts":"1731774936736"}
2024-11-16T16:35:36,739 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-16T16:35:36,740 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T16:35:36,743 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 821 msec
2024-11-16T16:35:37,289 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:35:37,292 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:37,312 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:37,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:37,314 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:41,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:35:41,189 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T16:35:41,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-16T16:35:41,190 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-16T16:35:41,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:35:41,191 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T16:35:42,204 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:35:42,206 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:42,228 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:42,231 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:42,231 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:35:42,241 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-16T16:35:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:35:46,017 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-16T16:35:46,017 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-16T16:35:46,021 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-16T16:35:46,021 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:46,034 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:46,038 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:46,039 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:46,039 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:46,039 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:35:46,039 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56161739{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:46,040 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e0d6f2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:46,154 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea87f36{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-41831-hadoop-hdfs-3_4_1-tests_jar-_-any-5971371034158736229/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:46,154 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@45b83ed{HTTP/1.1, (http/1.1)}{localhost:41831}
2024-11-16T16:35:46,154 INFO  [Time-limited test {}] server.Server(415): Started @121930ms
2024-11-16T16:35:46,156 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:46,187 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:46,190 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:46,191 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:46,191 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:46,191 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:35:46,192 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7645500{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:46,192 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70c175e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:46,301 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@592c6d7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-45037-hadoop-hdfs-3_4_1-tests_jar-_-any-14151802729526797640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:46,302 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1d39bcd7{HTTP/1.1, (http/1.1)}{localhost:45037}
2024-11-16T16:35:46,302 INFO  [Time-limited test {}] server.Server(415): Started @122077ms
2024-11-16T16:35:46,303 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:46,378 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:35:46,383 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:35:46,384 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:35:46,384 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:35:46,384 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:35:46,388 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@c4a6f04{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:35:46,389 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@300b140{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:35:46,505 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59934ee0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-36857-hadoop-hdfs-3_4_1-tests_jar-_-any-2498108889476579483/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:46,505 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a60480b{HTTP/1.1, (http/1.1)}{localhost:36857}
2024-11-16T16:35:46,505 INFO  [Time-limited test {}] server.Server(415): Started @122281ms
2024-11-16T16:35:46,507 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:35:47,612 WARN  [Thread-868 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data5/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:47,613 WARN  [Thread-869 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data6/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:47,634 WARN  [Thread-810 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:47,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x897172cd82bdce3b with lease ID 0xa651c37709922914: Processing first storage report for DS-30874b60-4bcd-474e-a004-223261332fe2 from datanode DatanodeRegistration(127.0.0.1:33309, datanodeUuid=6121d7b4-3192-4373-9d4b-93d6a085b8c5, infoPort=37841, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:47,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x897172cd82bdce3b with lease ID 0xa651c37709922914: from storage DS-30874b60-4bcd-474e-a004-223261332fe2 node DatanodeRegistration(127.0.0.1:33309, datanodeUuid=6121d7b4-3192-4373-9d4b-93d6a085b8c5, infoPort=37841, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:47,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x897172cd82bdce3b with lease ID 0xa651c37709922914: Processing first storage report for DS-4c9893ae-e132-4499-9588-47ea1fff82c5 from datanode DatanodeRegistration(127.0.0.1:33309, datanodeUuid=6121d7b4-3192-4373-9d4b-93d6a085b8c5, infoPort=37841, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:47,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x897172cd82bdce3b with lease ID 0xa651c37709922914: from storage DS-4c9893ae-e132-4499-9588-47ea1fff82c5 node DatanodeRegistration(127.0.0.1:33309, datanodeUuid=6121d7b4-3192-4373-9d4b-93d6a085b8c5, infoPort=37841, infoSecurePort=0, ipcPort=39823, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:47,856 WARN  [Thread-880 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:47,856 WARN  [Thread-881 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:47,879 WARN  [Thread-832 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:47,884 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56e35693b385307d with lease ID 0xa651c37709922915: Processing first storage report for DS-f0d181d1-bb57-4700-a774-b0248ce71325 from datanode DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:47,884 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56e35693b385307d with lease ID 0xa651c37709922915: from storage DS-f0d181d1-bb57-4700-a774-b0248ce71325 node DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:47,885 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x56e35693b385307d with lease ID 0xa651c37709922915: Processing first storage report for DS-a79067bd-54fe-48d6-88c1-544e24a741b5 from datanode DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:47,885 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x56e35693b385307d with lease ID 0xa651c37709922915: from storage DS-a79067bd-54fe-48d6-88c1-544e24a741b5 node DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:47,987 WARN  [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data9/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:47,987 WARN  [Thread-892 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data10/current/BP-2075037832-172.17.0.2-1731774932374/current, will proceed with Du for space computation calculation, 
2024-11-16T16:35:48,007 WARN  [Thread-854 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:35:48,009 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc41d87a393387fff with lease ID 0xa651c37709922916: Processing first storage report for DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d from datanode DatanodeRegistration(127.0.0.1:43277, datanodeUuid=47d21f88-bf65-464d-a98d-5431eca39dc4, infoPort=42961, infoSecurePort=0, ipcPort=42305, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:48,009 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc41d87a393387fff with lease ID 0xa651c37709922916: from storage DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d node DatanodeRegistration(127.0.0.1:43277, datanodeUuid=47d21f88-bf65-464d-a98d-5431eca39dc4, infoPort=42961, infoSecurePort=0, ipcPort=42305, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:48,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc41d87a393387fff with lease ID 0xa651c37709922916: Processing first storage report for DS-d9fd680c-4c2c-480d-91f2-20acc063d350 from datanode DatanodeRegistration(127.0.0.1:43277, datanodeUuid=47d21f88-bf65-464d-a98d-5431eca39dc4, infoPort=42961, infoSecurePort=0, ipcPort=42305, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374)
2024-11-16T16:35:48,010 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc41d87a393387fff with lease ID 0xa651c37709922916: from storage DS-d9fd680c-4c2c-480d-91f2-20acc063d350 node DatanodeRegistration(127.0.0.1:43277, datanodeUuid=47d21f88-bf65-464d-a98d-5431eca39dc4, infoPort=42961, infoSecurePort=0, ipcPort=42305, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:35:48,056 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,056 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012
java.io.IOException: Bad response ERROR for BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012 from datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,057 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:48,057 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:48,057 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006
java.io.IOException: Bad response ERROR for BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,057 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:48,057 WARN  [PacketResponder: BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42837] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,058 WARN  [PacketResponder: BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42837] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:51420 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:41585:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51420 dst: /127.0.0.1:41585
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116582920_22 at /127.0.0.1:51364 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41585:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51364 dst: /127.0.0.1:41585
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,059 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:54662 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42837:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54662 dst: /127.0.0.1:42837
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116582920_22 at /127.0.0.1:54626 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42837:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54626 dst: /127.0.0.1:42837
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,058 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:54690 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012] {}] datanode.DataXceiver(331): 127.0.0.1:42837:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54690 dst: /127.0.0.1:42837
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@19a6c3ca{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:48,061 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e259212{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:48,061 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:48,061 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@630b1d87{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:48,061 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,062 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f400306{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:48,062 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta block BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:48,062 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:51382 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41585:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51382 dst: /127.0.0.1:41585
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,062 WARN  [PacketResponder: BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:42837] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,063 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:48,063 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:48,063 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid e28f5a31-d7b1-4830-abef-9e7c86b1febc) service to localhost/127.0.0.1:36911
2024-11-16T16:35:48,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:54668 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42837:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54668 dst: /127.0.0.1:42837
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[closed]. Total timeout mills is 60000, 48675 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,063 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:48,063 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:51394 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41585:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:51394 dst: /127.0.0.1:41585
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data3/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:48,064 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data4/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:48,064 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:48,065 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@550a8a10 {}] datanode.DataXceiver(331): 127.0.0.1:41585:DataXceiver error processing unknown operation  src: /127.0.0.1:57370 dst: /127.0.0.1:41585
java.io.IOException: Server closed.
	at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:48,065 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,066 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,066 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta block BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,068 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741836_1012
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1705bda4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:48,069 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2cbd3715{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:48,069 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:48,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@149f32b8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:48,069 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@357bf795{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:48,071 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:48,071 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:48,071 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:48,071 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid 78d4b138-0b42-4053-b5f7-f4f9e469c507) service to localhost/127.0.0.1:36911
2024-11-16T16:35:48,071 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data1/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:48,072 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data2/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:48,072 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:48,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., hostname=303fe01965e8,33539,1731774935781, seqNum=2]
2024-11-16T16:35:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:35:48,079 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:35:48,080 ERROR [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4-prefix:303fe01965e8,33539,1731774935781 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,081 WARN  [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4-prefix:303fe01965e8,33539,1731774935781 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,081 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C33539%2C1731774935781:(num 1731774936019) roll requested
2024-11-16T16:35:48,081 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774948081
2024-11-16T16:35:48,087 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:48,087 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:48,087 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:48,087 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:48,087 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:48,088 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081
2024-11-16T16:35:48,088 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,088 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:48,089 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42961:42961),(127.0.0.1/127.0.0.1:37841:37841)]
2024-11-16T16:35:48,089 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:48,089 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-16T16:35:48,090 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-16T16:35:48,090 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019
2024-11-16T16:35:48,093 WARN  [IPC Server handler 2 on default port 36911 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741836_1012
2024-11-16T16:35:48,097 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 after 4ms
2024-11-16T16:35:48,753 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:49,118 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:49,119 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:50,091 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081
2024-11-16T16:35:50,092 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:50,092 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:50,092 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:54880 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:43277:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54880 dst: /127.0.0.1:43277
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:50,093 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:58272 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:33309:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58272 dst: /127.0.0.1:33309
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:50,131 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59934ee0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:50,131 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a60480b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:50,131 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:50,132 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@300b140{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:50,132 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@c4a6f04{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:50,133 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:50,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:50,133 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid 47d21f88-bf65-464d-a98d-5431eca39dc4) service to localhost/127.0.0.1:36911
2024-11-16T16:35:50,133 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:50,134 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data9/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:50,134 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data10/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:50,134 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:50,753 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:51,119 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:51,119 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:52,089 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]
2024-11-16T16:35:52,090 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C33539%2C1731774935781:(num 1731774948081) roll requested
2024-11-16T16:35:52,090 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774952090
2024-11-16T16:35:52,094 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42837
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:52,094 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41712 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021 to mirror 127.0.0.1:42837
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:52,094 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:52,095 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021
2024-11-16T16:35:52,095 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41712 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021] {}] datanode.BlockReceiver(316): Block 1073741839 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:52,095 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41712 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41712 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:52,097 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:52,097 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 after 4007ms
2024-11-16T16:35:52,101 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:52,101 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:52,101 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741840_1022
2024-11-16T16:35:52,101 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:52,104 WARN  [Thread-912 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43277
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:52,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:55576 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data6]'}, localName='127.0.0.1:33309', datanodeUuid='6121d7b4-3192-4373-9d4b-93d6a085b8c5', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023 to mirror 127.0.0.1:43277
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:52,104 WARN  [Thread-912 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:52,104 WARN  [Thread-912 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023
2024-11-16T16:35:52,104 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:55576 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:52,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:55576 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:33309:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55576 dst: /127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:52,105 WARN  [Thread-912 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:52,109 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:52,109 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:52,109 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:52,109 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:52,109 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:52,110 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774952090
2024-11-16T16:35:52,112 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45339:45339),(127.0.0.1/127.0.0.1:37841:37841)]
2024-11-16T16:35:52,112 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:52,112 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081 is not closed yet, will try archiving it next time
2024-11-16T16:35:52,113 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33309 is added to blk_1073741838_1020 (size=2431)
2024-11-16T16:35:52,139 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-16T16:35:52,514 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:52,754 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:53,120 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:53,120 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,143 WARN  [ResponseProcessor for block BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024
java.io.IOException: Bad response ERROR for BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024 from datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,144 WARN  [DataStreamer for file /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774952090 block BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:54,144 WARN  [PacketResponder: BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33309] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41726 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41726 dst: /127.0.0.1:37865
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,145 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:55582 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:33309:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:55582 dst: /127.0.0.1:33309
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,183 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea87f36{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:35:54,184 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@45b83ed{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:35:54,184 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:35:54,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e0d6f2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:35:54,184 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56161739{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:35:54,186 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:35:54,186 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:35:54,186 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid 6121d7b4-3192-4373-9d4b-93d6a085b8c5) service to localhost/127.0.0.1:36911
2024-11-16T16:35:54,186 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:35:54,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data5/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:54,187 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data6/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:35:54,188 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:35:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:54,202 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:35:54,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/3f9ec25a940544a7ba54b92c97117b4f is 1080, key is row0002/info:/1731774950136/Put/seqid=0
2024-11-16T16:35:54,227 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,227 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:54,227 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741843_1026
2024-11-16T16:35:54,227 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:54,229 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,229 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:54,229 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741844_1027
2024-11-16T16:35:54,230 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:54,231 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,231 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:54,231 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741845_1028
2024-11-16T16:35:54,232 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:54,234 WARN  [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,234 WARN  [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:54,234 WARN  [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741846_1029
2024-11-16T16:35:54,234 WARN  [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:54,235 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:54,235 WARN  [IPC Server handler 3 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:54,236 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:54,239 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741847_1030 (size=10347)
2024-11-16T16:35:54,640 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/3f9ec25a940544a7ba54b92c97117b4f
2024-11-16T16:35:54,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/3f9ec25a940544a7ba54b92c97117b4f as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f
2024-11-16T16:35:54,658 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f, entries=5, sequenceid=11, filesize=10.1 K
2024-11-16T16:35:54,659 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 3eddc71f8121fc89cc96f04482c1d1c1 in 457ms, sequenceid=11, compaction requested=false
2024-11-16T16:35:54,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:54,754 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:54,829 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-16T16:35:54,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/1489f837829b463490c463d92b859b0f is 1080, key is row0007/info:/1731774954204/Put/seqid=0
2024-11-16T16:35:54,837 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42837
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,837 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41750 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031 to mirror 127.0.0.1:42837
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,837 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:54,838 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031
2024-11-16T16:35:54,838 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41750 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:54,838 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41750 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41750 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,838 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:54,840 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,840 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:54,840 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741849_1032
2024-11-16T16:35:54,841 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:54,842 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,843 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:54,843 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741850_1033
2024-11-16T16:35:54,843 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:54,846 WARN  [Thread-928 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:54,846 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41762 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,846 WARN  [Thread-928 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:54,846 WARN  [Thread-928 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034
2024-11-16T16:35:54,846 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41762 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:54,846 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41762 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41762 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:54,846 WARN  [Thread-928 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:54,847 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:54,847 WARN  [IPC Server handler 3 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:54,847 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:54,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741852_1035 (size=12506)
2024-11-16T16:35:55,120 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:55,120 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:55,251 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/1489f837829b463490c463d92b859b0f
2024-11-16T16:35:55,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/1489f837829b463490c463d92b859b0f as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f
2024-11-16T16:35:55,266 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f, entries=7, sequenceid=24, filesize=12.2 K
2024-11-16T16:35:55,267 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 3eddc71f8121fc89cc96f04482c1d1c1 in 438ms, sequenceid=24, compaction requested=false
2024-11-16T16:35:55,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:55,267 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-16T16:35:55,267 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:55,267 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f because midkey is the same as first or last row
2024-11-16T16:35:56,112 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]
2024-11-16T16:35:56,112 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C33539%2C1731774935781:(num 1731774952090) roll requested
2024-11-16T16:35:56,113 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774956112
2024-11-16T16:35:56,116 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,116 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:56,116 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741853_1036
2024-11-16T16:35:56,117 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:56,118 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,118 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:56,118 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741854_1037
2024-11-16T16:35:56,119 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:56,120 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,120 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:56,120 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741855_1038
2024-11-16T16:35:56,121 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:56,122 WARN  [Thread-935 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,122 WARN  [Thread-935 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:56,122 WARN  [Thread-935 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741856_1039
2024-11-16T16:35:56,123 WARN  [Thread-935 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:56,123 WARN  [IPC Server handler 4 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:56,123 WARN  [IPC Server handler 4 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:56,124 WARN  [IPC Server handler 4 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:56,126 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:56,127 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:56,127 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:56,127 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:56,127 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:56,127 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774952090 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774956112
2024-11-16T16:35:56,128 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45339:45339)]
2024-11-16T16:35:56,128 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:56,129 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774952090 is not closed yet, will try archiving it next time
2024-11-16T16:35:56,129 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774948081 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs/303fe01965e8%2C33539%2C1731774935781.1731774948081
2024-11-16T16:35:56,132 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741842_1025 (size=25992)
2024-11-16T16:35:56,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:56,249 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-16T16:35:56,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/696dc838754742cc9d279f24762e25b1 is 1079, key is tmprow/info:/1731774956247/Put/seqid=0
2024-11-16T16:35:56,258 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,258 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:56,258 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741858_1041
2024-11-16T16:35:56,259 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:56,260 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,261 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:56,261 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741859_1042
2024-11-16T16:35:56,261 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:56,263 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,263 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:56,263 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741860_1043
2024-11-16T16:35:56,264 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:56,265 WARN  [Thread-939 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,265 WARN  [Thread-939 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:56,265 WARN  [Thread-939 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741861_1044
2024-11-16T16:35:56,266 WARN  [Thread-939 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:56,267 WARN  [IPC Server handler 1 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:56,267 WARN  [IPC Server handler 1 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:56,267 WARN  [IPC Server handler 1 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:56,270 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741862_1045 (size=6027)
2024-11-16T16:35:56,531 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:56,671 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/696dc838754742cc9d279f24762e25b1
2024-11-16T16:35:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/696dc838754742cc9d279f24762e25b1 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1
2024-11-16T16:35:56,689 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1, entries=1, sequenceid=34, filesize=5.9 K
2024-11-16T16:35:56,690 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 3eddc71f8121fc89cc96f04482c1d1c1 in 441ms, sequenceid=34, compaction requested=true
2024-11-16T16:35:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-16T16:35:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:56,690 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f because midkey is the same as first or last row
2024-11-16T16:35:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3eddc71f8121fc89cc96f04482c1d1c1:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:35:56,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:56,692 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:35:56,694 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:35:56,694 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1541): 3eddc71f8121fc89cc96f04482c1d1c1/info is initiating minor compaction (all files)
2024-11-16T16:35:56,694 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3eddc71f8121fc89cc96f04482c1d1c1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:56,695 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1] into tmpdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp, totalSize=28.2 K
2024-11-16T16:35:56,695 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f9ec25a940544a7ba54b92c97117b4f, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731774950136
2024-11-16T16:35:56,696 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1489f837829b463490c463d92b859b0f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731774954204
2024-11-16T16:35:56,696 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 696dc838754742cc9d279f24762e25b1, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731774956247
2024-11-16T16:35:56,714 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3eddc71f8121fc89cc96f04482c1d1c1#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:35:56,715 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/03365e0348e94d728ecc27886d3a0667 is 1080, key is row0002/info:/1731774950136/Put/seqid=0
2024-11-16T16:35:56,717 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,717 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:56,717 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741863_1046
2024-11-16T16:35:56,718 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:56,719 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,719 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:56,720 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741864_1047
2024-11-16T16:35:56,720 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:56,722 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41585
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41818 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048 to mirror 127.0.0.1:41585
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:56,723 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:56,723 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048
2024-11-16T16:35:56,723 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41818 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:56,723 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41818 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41818 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:56,723 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:56,726 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:56,726 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41822 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:56,726 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:56,726 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41822 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:56,726 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049
2024-11-16T16:35:56,726 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41822 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41822 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:56,727 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:56,727 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:56,727 WARN  [IPC Server handler 3 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:56,727 WARN  [IPC Server handler 3 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:56,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741867_1050 (size=17994)
2024-11-16T16:35:56,755 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,121 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,121 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,142 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/03365e0348e94d728ecc27886d3a0667 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667
2024-11-16T16:35:57,151 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3eddc71f8121fc89cc96f04482c1d1c1/info of 3eddc71f8121fc89cc96f04482c1d1c1 into 03365e0348e94d728ecc27886d3a0667(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:57,151 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., storeName=3eddc71f8121fc89cc96f04482c1d1c1/info, priority=13, startTime=1731774956691; duration=0sec
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 because midkey is the same as first or last row
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:57,151 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 because midkey is the same as first or last row
2024-11-16T16:35:57,152 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-16T16:35:57,152 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:57,152 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 because midkey is the same as first or last row
2024-11-16T16:35:57,152 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:57,152 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3eddc71f8121fc89cc96f04482c1d1c1:info
2024-11-16T16:35:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:57,670 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-16T16:35:57,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/0fc37ee49d7d4e28b1e48a095b3ae28b is 1079, key is tmprow/info:/1731774957669/Put/seqid=0
2024-11-16T16:35:57,677 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,677 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:57,677 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741868_1051
2024-11-16T16:35:57,678 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:57,679 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,679 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:57,680 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741869_1052
2024-11-16T16:35:57,680 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:57,682 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,682 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:57,682 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741870_1053
2024-11-16T16:35:57,682 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:57,684 WARN  [Thread-951 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:57,684 WARN  [Thread-951 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:57,684 WARN  [Thread-951 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741871_1054
2024-11-16T16:35:57,685 WARN  [Thread-951 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:57,685 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:57,685 WARN  [IPC Server handler 0 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:57,686 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:57,698 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741872_1055 (size=6027)
2024-11-16T16:35:57,895 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741847_1030 to 127.0.0.1:41585 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:57,895 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d00c855[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741852_1035 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,099 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/0fc37ee49d7d4e28b1e48a095b3ae28b
2024-11-16T16:35:58,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/0fc37ee49d7d4e28b1e48a095b3ae28b as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b
2024-11-16T16:35:58,115 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b, entries=1, sequenceid=45, filesize=5.9 K
2024-11-16T16:35:58,116 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 3eddc71f8121fc89cc96f04482c1d1c1 in 446ms, sequenceid=45, compaction requested=false
2024-11-16T16:35:58,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:58,116 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-16T16:35:58,117 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:58,117 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 because midkey is the same as first or last row
2024-11-16T16:35:58,129 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]
2024-11-16T16:35:58,129 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C33539%2C1731774935781:(num 1731774956112) roll requested
2024-11-16T16:35:58,130 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774958129
2024-11-16T16:35:58,134 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:43277
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:58,133 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41854 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056 to mirror 127.0.0.1:43277
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,134 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:58,134 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056
2024-11-16T16:35:58,134 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41854 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056] {}] datanode.BlockReceiver(316): Block 1073741873 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:58,134 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41854 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741873_1056] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41854 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,135 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:58,137 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:58,137 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41870 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,138 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:58,138 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057
2024-11-16T16:35:58,138 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41870 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:58,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41870 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41870 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,139 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:58,141 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41585
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:58,141 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41884 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058 to mirror 127.0.0.1:41585
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,141 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:58,141 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058
2024-11-16T16:35:58,141 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41884 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:58,141 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41884 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41884 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,142 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:58,144 WARN  [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42837
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:58,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41894 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059 to mirror 127.0.0.1:42837
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,144 WARN  [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:58,144 WARN  [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059
2024-11-16T16:35:58,144 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41894 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:35:58,144 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41894 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41894 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,145 WARN  [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:58,145 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:58,146 WARN  [IPC Server handler 0 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:58,146 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:58,148 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:58,148 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:58,148 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:58,148 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:58,148 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:35:58,148 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774956112 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774958129
2024-11-16T16:35:58,149 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45339:45339)]
2024-11-16T16:35:58,149 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:58,149 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774956112 is not closed yet, will try archiving it next time
2024-11-16T16:35:58,149 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774952090 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs/303fe01965e8%2C33539%2C1731774935781.1731774952090
2024-11-16T16:35:58,150 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741857_1040 (size=13591)
2024-11-16T16:35:58,551 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 is not closed yet, will try archiving it next time
2024-11-16T16:35:58,755 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:58,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d00c855[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741862_1045 to 127.0.0.1:42837 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:58,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741842_1025 to 127.0.0.1:41585 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:35:59,092 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-16T16:35:59,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/6083ae0ff3654c5e8e03d8be2e7c8134 is 1079, key is tmprow/info:/1731774959091/Put/seqid=0
2024-11-16T16:35:59,099 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,100 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:59,100 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741878_1061
2024-11-16T16:35:59,100 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:59,101 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:59,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741879_1062
2024-11-16T16:35:59,102 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:59,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41585
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41918 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063 to mirror 127.0.0.1:41585
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:59,104 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063
2024-11-16T16:35:59,105 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41918 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:59,105 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41918 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41918 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,105 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:59,109 WARN  [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42837
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,109 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41924 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064 to mirror 127.0.0.1:42837
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,110 WARN  [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:59,110 WARN  [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064
2024-11-16T16:35:59,110 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41924 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:59,110 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41924 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41924 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,110 WARN  [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:59,111 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:59,112 WARN  [IPC Server handler 0 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:59,112 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:59,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741882_1065 (size=6027)
2024-11-16T16:35:59,121 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,121 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,516 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/6083ae0ff3654c5e8e03d8be2e7c8134
2024-11-16T16:35:59,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/6083ae0ff3654c5e8e03d8be2e7c8134 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134
2024-11-16T16:35:59,530 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134, entries=1, sequenceid=55, filesize=5.9 K
2024-11-16T16:35:59,531 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 3eddc71f8121fc89cc96f04482c1d1c1 in 439ms, sequenceid=55, compaction requested=true
2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 because midkey is the same as first or last row
2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3eddc71f8121fc89cc96f04482c1d1c1:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:35:59,532 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:35:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:59,534 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:35:59,534 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1541): 3eddc71f8121fc89cc96f04482c1d1c1/info is initiating minor compaction (all files)
2024-11-16T16:35:59,534 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3eddc71f8121fc89cc96f04482c1d1c1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:35:59,534 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134] into tmpdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp, totalSize=29.3 K
2024-11-16T16:35:59,535 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03365e0348e94d728ecc27886d3a0667, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731774950136
2024-11-16T16:35:59,535 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc37ee49d7d4e28b1e48a095b3ae28b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731774957669
2024-11-16T16:35:59,536 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6083ae0ff3654c5e8e03d8be2e7c8134, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731774959091
2024-11-16T16:35:59,552 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3eddc71f8121fc89cc96f04482c1d1c1#info#compaction#24 average throughput is 12.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:35:59,553 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a5304b572f1b476c801da57886b62325 is 1080, key is row0002/info:/1731774950136/Put/seqid=0
2024-11-16T16:35:59,555 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,555 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]) is bad.
2024-11-16T16:35:59,555 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741883_1066
2024-11-16T16:35:59,556 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]
2024-11-16T16:35:59,557 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,557 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]) is bad.
2024-11-16T16:35:59,558 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741884_1067
2024-11-16T16:35:59,558 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:43277,DS-dfc2ae14-fc7e-45e2-883a-5c0458b9dc6d,DISK]
2024-11-16T16:35:59,560 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:42837
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,560 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41936 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068 to mirror 127.0.0.1:42837
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,561 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]) is bad.
2024-11-16T16:35:59,561 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068
2024-11-16T16:35:59,561 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41936 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:35:59,561 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:41936 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741885_1068] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41936 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:35:59,561 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:42837,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK]
2024-11-16T16:35:59,563 WARN  [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:35:59,563 WARN  [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:35:59,563 WARN  [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741886_1069
2024-11-16T16:35:59,564 WARN  [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:35:59,565 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-16T16:35:59,565 WARN  [IPC Server handler 0 on default port 36911 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-16T16:35:59,565 WARN  [IPC Server handler 0 on default port 36911 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-16T16:35:59,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741887_1070 (size=18097)
2024-11-16T16:35:59,980 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a5304b572f1b476c801da57886b62325 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325
2024-11-16T16:35:59,989 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3eddc71f8121fc89cc96f04482c1d1c1/info of 3eddc71f8121fc89cc96f04482c1d1c1 into a5304b572f1b476c801da57886b62325(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:35:59,989 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:35:59,989 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., storeName=3eddc71f8121fc89cc96f04482c1d1c1/info, priority=13, startTime=1731774959532; duration=0sec
2024-11-16T16:35:59,989 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-16T16:35:59,989 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 because midkey is the same as first or last row
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 because midkey is the same as first or last row
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 because midkey is the same as first or last row
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:35:59,990 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3eddc71f8121fc89cc96f04482c1d1c1:info
2024-11-16T16:36:00,150 WARN  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-16T16:36:00,318 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:00,322 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:00,323 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:00,323 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:00,323 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:36:00,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7cac94da{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:00,324 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@278e888c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:00,429 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2817e38a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/java.io.tmpdir/jetty-localhost-44825-hadoop-hdfs-3_4_1-tests_jar-_-any-8191865413474974940/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:00,430 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f9cc2c{HTTP/1.1, (http/1.1)}{localhost:44825}
2024-11-16T16:36:00,430 INFO  [Time-limited test {}] server.Server(415): Started @136205ms
2024-11-16T16:36:00,431 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:00,756 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:00,873 WARN  [Thread-991 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:00,880 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3fae6a457be1a4f with lease ID 0xa651c37709922917: from storage DS-7da5a248-5b7c-4cd4-840d-4b120725ae67 node DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:00,881 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb3fae6a457be1a4f with lease ID 0xa651c37709922917: from storage DS-5460f134-4094-4b7c-8c9d-3a245d22089a node DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:00,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d00c855[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741872_1055 to 127.0.0.1:43277 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:00,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741867_1050 to 127.0.0.1:43277 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:01,122 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:01,122 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:01,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741857_1040 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:01,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4d00c855[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741882_1065 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:02,756 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:03,122 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:03,123 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:03,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741887_1070 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:04,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:36:04,756 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:04,994 ERROR [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData-prefix:303fe01965e8,39189,1731774934458 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:04,994 WARN  [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData-prefix:303fe01965e8,39189,1731774934458 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:04,994 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C39189%2C1731774934458:(num 1731774934774) roll requested
2024-11-16T16:36:04,995 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C39189%2C1731774934458.1731774964994
2024-11-16T16:36:05,001 WARN  [Thread-1011 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:05,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116582920_22 at /127.0.0.1:42826 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data4]'}, localName='127.0.0.1:38241', datanodeUuid='e28f5a31-d7b1-4830-abef-9e7c86b1febc', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:05,001 WARN  [Thread-1011 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38241,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:05,001 WARN  [Thread-1011 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071
2024-11-16T16:36:05,001 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1116582920_22 at /127.0.0.1:42826 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-16T16:36:05,001 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1116582920_22 at /127.0.0.1:42826 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741888_1071] {}] datanode.DataXceiver(331): 127.0.0.1:38241:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42826 dst: /127.0.0.1:38241
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:05,002 WARN  [Thread-1011 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:05,005 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:05,005 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:05,006 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:05,006 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:05,006 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:05,006 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774964994
2024-11-16T16:36:05,006 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:05,007 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:05,007 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774
2024-11-16T16:36:05,007 WARN  [IPC Server handler 0 on default port 36911 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-11-16T16:36:05,007 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 after 0ms
2024-11-16T16:36:05,009 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45339:45339),(127.0.0.1/127.0.0.1:34111:34111)]
2024-11-16T16:36:05,009 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 is not closed yet, will try archiving it next time
2024-11-16T16:36:05,123 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:05,123 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:07,123 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:07,124 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:09,008 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/WALs/303fe01965e8,39189,1731774934458/303fe01965e8%2C39189%2C1731774934458.1731774934774 after 4001ms
2024-11-16T16:36:09,124 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:09,124 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:09,879 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@644364ea[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741837_1013 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:09,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741835_1011 (size=393)
2024-11-16T16:36:10,878 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@273f7aba[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741831_1007 to 127.0.0.1:43277 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:10,880 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:36:10,894 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@471085f5 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741836_1012, datanode=DatanodeInfoWithStorage[127.0.0.1:41585,null,null])
java.net.ConnectException: Call From 303fe01965e8/172.17.0.2 to localhost:46543 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-16T16:36:10,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741836_1019 (size=455)
2024-11-16T16:36:11,116 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774936019 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs/303fe01965e8%2C33539%2C1731774935781.1731774936019
2024-11-16T16:36:11,125 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:11,125 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:11,129 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774956112 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs/303fe01965e8%2C33539%2C1731774935781.1731774956112
2024-11-16T16:36:12,879 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@644364ea[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741826_1002 to 127.0.0.1:43277 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:12,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741836_1019 (size=455)
2024-11-16T16:36:13,125 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:13,126 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:13,661 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C33539%2C1731774935781.1731774973660
2024-11-16T16:36:13,666 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:13,666 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:13,666 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:13,667 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:13,667 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:13,667 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774958129 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774973660
2024-11-16T16:36:13,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34111:34111),(127.0.0.1/127.0.0.1:45339:45339)]
2024-11-16T16:36:13,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774958129 is not closed yet, will try archiving it next time
2024-11-16T16:36:13,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741877_1060 (size=12911)
2024-11-16T16:36:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:36:13,674 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-16T16:36:13,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/5128c34b80374fe4a5c0f49e5f4ed31c is 1080, key is row0013/info:/1731774973669/Put/seqid=0
2024-11-16T16:36:13,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741891_1075 (size=8190)
2024-11-16T16:36:13,685 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741891_1075 (size=8190)
2024-11-16T16:36:13,686 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/5128c34b80374fe4a5c0f49e5f4ed31c
2024-11-16T16:36:13,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/5128c34b80374fe4a5c0f49e5f4ed31c as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c
2024-11-16T16:36:13,703 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c, entries=3, sequenceid=66, filesize=8.0 K
2024-11-16T16:36:13,704 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 3eddc71f8121fc89cc96f04482c1d1c1 in 31ms, sequenceid=66, compaction requested=false
2024-11-16T16:36:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:36:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-16T16:36:13,704 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:36:13,705 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 because midkey is the same as first or last row
2024-11-16T16:36:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33539 {}] regionserver.HRegion(8855): Flush requested on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:36:13,904 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-16T16:36:13,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a28c46c97e594fbeadf2273c31f98b68 is 1080, key is row0015/info:/1731774973675/Put/seqid=0
2024-11-16T16:36:13,913 WARN  [Thread-1041 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1076
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:13,913 WARN  [Thread-1041 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741892_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:13,914 WARN  [Thread-1041 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741892_1076
2024-11-16T16:36:13,914 WARN  [Thread-1041 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:13,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741893_1077 (size=14660)
2024-11-16T16:36:13,920 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741893_1077 (size=14660)
2024-11-16T16:36:13,921 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a28c46c97e594fbeadf2273c31f98b68
2024-11-16T16:36:13,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a28c46c97e594fbeadf2273c31f98b68 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68
2024-11-16T16:36:13,936 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68, entries=9, sequenceid=79, filesize=14.3 K
2024-11-16T16:36:13,937 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 3eddc71f8121fc89cc96f04482c1d1c1 in 34ms, sequenceid=79, compaction requested=true
2024-11-16T16:36:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:36:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-16T16:36:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:36:13,938 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 because midkey is the same as first or last row
2024-11-16T16:36:13,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3eddc71f8121fc89cc96f04482c1d1c1:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:36:13,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:36:13,938 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:36:13,939 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:36:13,939 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1541): 3eddc71f8121fc89cc96f04482c1d1c1/info is initiating minor compaction (all files)
2024-11-16T16:36:13,940 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3eddc71f8121fc89cc96f04482c1d1c1/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:13,940 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68] into tmpdir=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp, totalSize=40.0 K
2024-11-16T16:36:13,940 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5304b572f1b476c801da57886b62325, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731774950136
2024-11-16T16:36:13,941 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5128c34b80374fe4a5c0f49e5f4ed31c, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731774960106
2024-11-16T16:36:13,941 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28c46c97e594fbeadf2273c31f98b68, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731774973675
2024-11-16T16:36:13,958 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3eddc71f8121fc89cc96f04482c1d1c1#info#compaction#27 average throughput is 7.52 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:36:13,959 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/7a4ae0f907a74d168dc3b08353eec7e1 is 1080, key is row0002/info:/1731774950136/Put/seqid=0
2024-11-16T16:36:13,962 WARN  [Thread-1048 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:13,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:46056 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:13,962 WARN  [Thread-1048 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:13,962 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:46056 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078] {}] datanode.BlockReceiver(316): Block 1073741894 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:36:13,962 WARN  [Thread-1048 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078
2024-11-16T16:36:13,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1898174055_22 at /127.0.0.1:46056 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741894_1078] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46056 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:13,963 WARN  [Thread-1048 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:13,967 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741895_1079 (size=28989)
2024-11-16T16:36:13,968 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741895_1079 (size=28989)
2024-11-16T16:36:13,976 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/7a4ae0f907a74d168dc3b08353eec7e1 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/7a4ae0f907a74d168dc3b08353eec7e1
2024-11-16T16:36:13,985 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3eddc71f8121fc89cc96f04482c1d1c1/info of 3eddc71f8121fc89cc96f04482c1d1c1 into 7a4ae0f907a74d168dc3b08353eec7e1(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:36:13,985 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3eddc71f8121fc89cc96f04482c1d1c1:

2024-11-16T16:36:13,985 INFO  [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., storeName=3eddc71f8121fc89cc96f04482c1d1c1/info, priority=13, startTime=1731774973938; duration=0sec
2024-11-16T16:36:13,985 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-16T16:36:13,985 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:36:13,985 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/7a4ae0f907a74d168dc3b08353eec7e1 because midkey is the same as first or last row
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/7a4ae0f907a74d168dc3b08353eec7e1 because midkey is the same as first or last row
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/7a4ae0f907a74d168dc3b08353eec7e1 because midkey is the same as first or last row
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:36:13,986 DEBUG [RS:1;303fe01965e8:33539-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3eddc71f8121fc89cc96f04482c1d1c1:info
2024-11-16T16:36:14,070 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,33539,1731774935781/303fe01965e8%2C33539%2C1731774935781.1731774958129 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs/303fe01965e8%2C33539%2C1731774935781.1731774958129
2024-11-16T16:36:14,151 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-16T16:36:14,307 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:36:14,307 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:36:14,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:14,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,308 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:36:14,308 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:36:14,308 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=152599692, stopped=false
2024-11-16T16:36:14,308 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,39189,1731774934458
2024-11-16T16:36:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:14,401 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:14,401 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:14,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:14,402 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:36:14,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:14,402 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:36:14,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:14,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,403 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,36407,1731774934631' *****
2024-11-16T16:36:14,403 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:36:14,403 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,33539,1731774935781' *****
2024-11-16T16:36:14,403 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:36:14,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:14,404 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:36:14,404 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:36:14,404 INFO  [RS:1;303fe01965e8:33539 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:36:14,404 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:14,404 INFO  [RS:1;303fe01965e8:33539 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:36:14,404 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:36:14,405 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(3091): Received CLOSE for 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:36:14,405 INFO  [RS:0;303fe01965e8:36407 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:36:14,405 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:36:14,405 INFO  [RS:0;303fe01965e8:36407 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:36:14,405 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,36407,1731774934631
2024-11-16T16:36:14,405 INFO  [RS:0;303fe01965e8:36407 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:36:14,405 INFO  [RS:0;303fe01965e8:36407 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:36407.
2024-11-16T16:36:14,405 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,33539,1731774935781
2024-11-16T16:36:14,406 DEBUG [RS:0;303fe01965e8:36407 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:14,406 INFO  [RS:1;303fe01965e8:33539 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:36:14,406 DEBUG [RS:0;303fe01965e8:36407 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,406 INFO  [RS:1;303fe01965e8:33539 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;303fe01965e8:33539.
2024-11-16T16:36:14,406 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:36:14,406 DEBUG [RS:1;303fe01965e8:33539 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:14,406 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:36:14,406 DEBUG [RS:1;303fe01965e8:33539 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,406 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:36:14,406 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3eddc71f8121fc89cc96f04482c1d1c1, disabling compactions & flushes
2024-11-16T16:36:14,406 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:36:14,406 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-16T16:36:14,406 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:14,406 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1325): Online Regions={3eddc71f8121fc89cc96f04482c1d1c1=TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.}
2024-11-16T16:36:14,406 DEBUG [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1351): Waiting on 3eddc71f8121fc89cc96f04482c1d1c1
2024-11-16T16:36:14,406 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:14,406 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1. after waiting 0 ms
2024-11-16T16:36:14,407 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:14,407 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3eddc71f8121fc89cc96f04482c1d1c1 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:36:14,407 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-16T16:36:14,407 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-16T16:36:14,408 DEBUG [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T16:36:14,408 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:36:14,408 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:36:14,408 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:36:14,408 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:36:14,408 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:36:14,408 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-16T16:36:14,408 ERROR [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4-prefix:303fe01965e8,36407,1731774934631.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,409 WARN  [FSHLog-0-hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4-prefix:303fe01965e8,36407,1731774934631.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,409 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,409 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C36407%2C1731774934631.meta:.meta(num 1731774935631) roll requested
2024-11-16T16:36:14,409 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36407%2C1731774934631.meta.1731774974409.meta
2024-11-16T16:36:14,412 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a2918ccc415246079683c236ba8c4c69 is 1079, key is tmprow/info:/1731774974105/Put/seqid=0
2024-11-16T16:36:14,414 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,414 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK], DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:14,415 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741897_1081
2024-11-16T16:36:14,415 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:14,417 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,417 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,418 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,418 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,418 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,418 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774974409.meta
2024-11-16T16:36:14,425 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,425 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,425 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
2024-11-16T16:36:14,426 WARN  [IPC Server handler 3 on default port 36911 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta has not been closed. Lease recovery is in progress. RecoveryId = 1083 for block blk_1073741834_1010
2024-11-16T16:36:14,426 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta after 1ms
2024-11-16T16:36:14,430 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45339:45339),(127.0.0.1/127.0.0.1:34111:34111)]
2024-11-16T16:36:14,430 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta is not closed yet, will try archiving it next time
2024-11-16T16:36:14,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741898_1082 (size=6027)
2024-11-16T16:36:14,431 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741898_1082 (size=6027)
2024-11-16T16:36:14,431 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a2918ccc415246079683c236ba8c4c69
2024-11-16T16:36:14,439 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/.tmp/info/a2918ccc415246079683c236ba8c4c69 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a2918ccc415246079683c236ba8c4c69
2024-11-16T16:36:14,447 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a2918ccc415246079683c236ba8c4c69, entries=1, sequenceid=84, filesize=5.9 K
2024-11-16T16:36:14,448 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3eddc71f8121fc89cc96f04482c1d1c1 in 41ms, sequenceid=84, compaction requested=false
2024-11-16T16:36:14,449 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68] to archive
2024-11-16T16:36:14,450 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/info/81a9577ad05240139c86168b854a7494 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1./info:regioninfo/1731774936723/Put/seqid=0
2024-11-16T16:36:14,450 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:36:14,453 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/3f9ec25a940544a7ba54b92c97117b4f
2024-11-16T16:36:14,453 WARN  [Thread-1067 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1084
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,453 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:40998 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data4]'}, localName='127.0.0.1:38241', datanodeUuid='e28f5a31-d7b1-4830-abef-9e7c86b1febc', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:14,453 WARN  [Thread-1067 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38241,DS-7da5a248-5b7c-4cd4-840d-4b120725ae67,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:14,453 WARN  [Thread-1067 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084
2024-11-16T16:36:14,453 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:40998 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084] {}] datanode.BlockReceiver(316): Block 1073741899 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:36:14,453 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:40998 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741899_1084] {}] datanode.DataXceiver(331): 127.0.0.1:38241:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:40998 dst: /127.0.0.1:38241
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:14,454 WARN  [Thread-1067 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:14,455 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/1489f837829b463490c463d92b859b0f
2024-11-16T16:36:14,456 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/03365e0348e94d728ecc27886d3a0667
2024-11-16T16:36:14,458 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/696dc838754742cc9d279f24762e25b1
2024-11-16T16:36:14,459 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/0fc37ee49d7d4e28b1e48a095b3ae28b
2024-11-16T16:36:14,461 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a5304b572f1b476c801da57886b62325
2024-11-16T16:36:14,462 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/6083ae0ff3654c5e8e03d8be2e7c8134
2024-11-16T16:36:14,463 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/5128c34b80374fe4a5c0f49e5f4ed31c
2024-11-16T16:36:14,465 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68 to hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/info/a28c46c97e594fbeadf2273c31f98b68
2024-11-16T16:36:14,466 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=303fe01965e8:39189 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-16T16:36:14,466 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [3f9ec25a940544a7ba54b92c97117b4f=10347, 1489f837829b463490c463d92b859b0f=12506, 03365e0348e94d728ecc27886d3a0667=17994, 696dc838754742cc9d279f24762e25b1=6027, 0fc37ee49d7d4e28b1e48a095b3ae28b=6027, a5304b572f1b476c801da57886b62325=18097, 6083ae0ff3654c5e8e03d8be2e7c8134=6027, 5128c34b80374fe4a5c0f49e5f4ed31c=8190, a28c46c97e594fbeadf2273c31f98b68=14660]
2024-11-16T16:36:14,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741900_1085 (size=7089)
2024-11-16T16:36:14,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741900_1085 (size=7089)
2024-11-16T16:36:14,469 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/info/81a9577ad05240139c86168b854a7494
2024-11-16T16:36:14,471 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/default/TestLogRolling-testLogRollOnDatanodeDeath/3eddc71f8121fc89cc96f04482c1d1c1/recovered.edits/87.seqid, newMaxSeqId=87, maxSeqId=1
2024-11-16T16:36:14,471 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:14,471 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3eddc71f8121fc89cc96f04482c1d1c1:
Waiting for close lock at 1731774974406Running coprocessor pre-close hooks at 1731774974406Disabling compacts and flushes for region at 1731774974406Disabling writes for close at 1731774974406Obtaining lock to block concurrent updates at 1731774974407 (+1 ms)Preparing flush snapshotting stores in 3eddc71f8121fc89cc96f04482c1d1c1 at 1731774974407Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731774974407Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1. at 1731774974408 (+1 ms)Flushing 3eddc71f8121fc89cc96f04482c1d1c1/info: creating writer at 1731774974408Flushing 3eddc71f8121fc89cc96f04482c1d1c1/info: appending metadata at 1731774974412 (+4 ms)Flushing 3eddc71f8121fc89cc96f04482c1d1c1/info: closing flushed file at 1731774974412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d556dc4: reopening flushed file at 1731774974438 (+26 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 3eddc71f8121fc89cc96f04482c1d1c1 in 41ms, sequenceid=84, compaction requested=false at 1731774974448 (+10 ms)Writing region close event to WAL at 1731774974467 (+19 ms)Running coprocessor post-close hooks at 1731774974471 (+4 ms)Closed at 1731774974471
2024-11-16T16:36:14,472 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731774935918.3eddc71f8121fc89cc96f04482c1d1c1.
2024-11-16T16:36:14,495 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/ns/1131405dcd9041cc9c19dc575f81d8ec is 43, key is default/ns:d/1731774935727/Put/seqid=0
2024-11-16T16:36:14,500 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741901_1086 (size=5153)
2024-11-16T16:36:14,501 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741901_1086 (size=5153)
2024-11-16T16:36:14,501 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/ns/1131405dcd9041cc9c19dc575f81d8ec
2024-11-16T16:36:14,525 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/table/4b617a8f4ee148a1bf48899e62b16ac5 is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731774936736/Put/seqid=0
2024-11-16T16:36:14,528 WARN  [Thread-1081 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1087
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33309
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:14,528 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:46120 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8]'}, localName='127.0.0.1:37865', datanodeUuid='3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289', xmitsInProgress=0}:Exception transferring block BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087 to mirror 127.0.0.1:33309
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:14,529 WARN  [Thread-1081 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087 in pipeline [DatanodeInfoWithStorage[127.0.0.1:37865,DS-f0d181d1-bb57-4700-a774-b0248ce71325,DISK], DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]) is bad.
2024-11-16T16:36:14,529 WARN  [Thread-1081 {}] hdfs.DataStreamer(1850): Abandoning BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087
2024-11-16T16:36:14,529 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:46120 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087] {}] datanode.BlockReceiver(316): Block 1073741902 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-16T16:36:14,529 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1324861396_22 at /127.0.0.1:46120 [Receiving block BP-2075037832-172.17.0.2-1731774932374:blk_1073741902_1087] {}] datanode.DataXceiver(331): 127.0.0.1:37865:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46120 dst: /127.0.0.1:37865
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:14,529 WARN  [Thread-1081 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33309,DS-30874b60-4bcd-474e-a004-223261332fe2,DISK]
2024-11-16T16:36:14,534 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741903_1088 (size=5424)
2024-11-16T16:36:14,535 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741903_1088 (size=5424)
2024-11-16T16:36:14,535 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/table/4b617a8f4ee148a1bf48899e62b16ac5
2024-11-16T16:36:14,542 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/info/81a9577ad05240139c86168b854a7494 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/info/81a9577ad05240139c86168b854a7494
2024-11-16T16:36:14,549 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/info/81a9577ad05240139c86168b854a7494, entries=10, sequenceid=11, filesize=6.9 K
2024-11-16T16:36:14,551 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/ns/1131405dcd9041cc9c19dc575f81d8ec as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/ns/1131405dcd9041cc9c19dc575f81d8ec
2024-11-16T16:36:14,558 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/ns/1131405dcd9041cc9c19dc575f81d8ec, entries=2, sequenceid=11, filesize=5.0 K
2024-11-16T16:36:14,559 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/.tmp/table/4b617a8f4ee148a1bf48899e62b16ac5 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/table/4b617a8f4ee148a1bf48899e62b16ac5
2024-11-16T16:36:14,566 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/table/4b617a8f4ee148a1bf48899e62b16ac5, entries=2, sequenceid=11, filesize=5.3 K
2024-11-16T16:36:14,568 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 160ms, sequenceid=11, compaction requested=false
2024-11-16T16:36:14,573 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-16T16:36:14,574 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:36:14,574 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:36:14,574 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774974408Running coprocessor pre-close hooks at 1731774974408Disabling compacts and flushes for region at 1731774974408Disabling writes for close at 1731774974408Obtaining lock to block concurrent updates at 1731774974408Preparing flush snapshotting stores in 1588230740 at 1731774974408Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731774974409 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731774974430 (+21 ms)Flushing 1588230740/info: creating writer at 1731774974430Flushing 1588230740/info: appending metadata at 1731774974449 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731774974449Flushing 1588230740/ns: creating writer at 1731774974476 (+27 ms)Flushing 1588230740/ns: appending metadata at 1731774974494 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731774974494Flushing 1588230740/table: creating writer at 1731774974508 (+14 ms)Flushing 1588230740/table: appending metadata at 1731774974525 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731774974525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c880650: reopening flushed file at 1731774974541 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21fb7272: reopening flushed file at 1731774974549 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a7083b1: reopening flushed file at 1731774974558 (+9 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 160ms, sequenceid=11, compaction requested=false at 1731774974568 (+10 ms)Writing region close event to WAL at 1731774974569 (+1 ms)Running coprocessor post-close hooks at 1731774974574 (+5 ms)Closed at 1731774974574
2024-11-16T16:36:14,574 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:36:14,606 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,33539,1731774935781; all regions closed.
2024-11-16T16:36:14,607 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,607 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,607 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,607 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,607 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,608 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,36407,1731774934631; all regions closed.
2024-11-16T16:36:14,608 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,608 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,608 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,608 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,608 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:14,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741890_1074 (size=18156)
2024-11-16T16:36:14,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741890_1074 (size=18156)
2024-11-16T16:36:14,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741896_1080 (size=825)
2024-11-16T16:36:14,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741896_1080 (size=825)
2024-11-16T16:36:14,613 DEBUG [RS:1;303fe01965e8:33539 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C33539%2C1731774935781:(num 1731774973660)
2024-11-16T16:36:14,613 DEBUG [RS:1;303fe01965e8:33539 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:36:14,613 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:36:14,613 INFO  [RS:1;303fe01965e8:33539 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:36:14,614 INFO  [RS:1;303fe01965e8:33539 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33539
2024-11-16T16:36:14,621 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,33539,1731774935781
2024-11-16T16:36:14,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:36:14,622 INFO  [RS:1;303fe01965e8:33539 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:36:14,622 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,33539,1731774935781]
2024-11-16T16:36:14,643 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,33539,1731774935781 already deleted, retry=false
2024-11-16T16:36:14,643 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,33539,1731774935781 expired; onlineServers=1
2024-11-16T16:36:14,732 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:14,732 INFO  [RS:1;303fe01965e8:33539 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:36:14,732 DEBUG [pool-381-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33539-0x101448069510002, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:14,732 INFO  [RS:1;303fe01965e8:33539 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,33539,1731774935781; zookeeper connection closed.
2024-11-16T16:36:14,733 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e68c173 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e68c173
2024-11-16T16:36:15,122 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:15,172 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T16:36:15,172 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T16:36:15,844 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-16T16:36:15,844 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-16T16:36:15,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:36:15,884 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:36:15,886 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@3b73d0ac[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:37865, datanodeUuid=3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289, infoPort=45339, infoSecurePort=0, ipcPort=39577, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741877_1060 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:15,888 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:16,430 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:16,879 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@273f7aba[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741828_1004 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:16,879 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@644364ea[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38241, datanodeUuid=e28f5a31-d7b1-4830-abef-9e7c86b1febc, infoPort=34111, infoSecurePort=0, ipcPort=39807, storageInfo=lv=-57;cid=testClusterID;nsid=2099995959;c=1731774932374):Failed to transfer BP-2075037832-172.17.0.2-1731774932374:blk_1073741832_1008 to 127.0.0.1:33309 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:18,427 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta after 4002ms
2024-11-16T16:36:18,431 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:19,472 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,496 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,497 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,498 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,517 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,518 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:19,609 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-16T16:36:19,614 DEBUG [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs
2024-11-16T16:36:19,614 INFO  [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C36407%2C1731774934631.meta:.meta(num 1731774974409)
2024-11-16T16:36:19,615 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:19,615 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:19,615 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:19,615 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:19,616 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:19,616 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:19,617 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:41585,DS-73e39b61-3448-4c69-af4d-56105bf53d27,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:19,617 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
2024-11-16T16:36:19,628 WARN  [IPC Server handler 0 on default port 36911 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 has not been closed. Lease recovery is in progress. RecoveryId = 1089 for block blk_1073741833_1009
2024-11-16T16:36:19,629 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 after 12ms
2024-11-16T16:36:20,020 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:36:20,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,039 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,039 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,043 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,045 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:20,431 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-16T16:36:20,433 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:20,898 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1c249adc {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:41585,null,null])
java.net.ConnectException: Call From 303fe01965e8/172.17.0.2 to localhost:46543 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-16T16:36:21,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-16T16:36:21,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:36:21,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:36:21,435 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:21,883 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741830_1073 (size=27318)
2024-11-16T16:36:22,431 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-16T16:36:22,437 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:23,439 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:23,630 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 after 4013ms
2024-11-16T16:36:24,432 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.FSHLog(580): 
java.nio.channels.ClosedChannelException: null
	at org.apache.hadoop.hdfs.ExceptionLastSeen.throwException4Close(ExceptionLastSeen.java:73) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.checkClosed(DFSOutputStream.java:158) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSOutputStream.getCurrentBlockReplication(DFSOutputStream.java:775) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.client.HdfsDataOutputStream.getCurrentBlockReplication(HdfsDataOutputStream.java:79) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.getLogReplication(FSHLog.java:577) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doCheckLogLowReplication(FSHLog.java:525) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.checkLogLowReplication(AbstractFSWAL.java:2224) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.checkLowReplication(AbstractWALRoller.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:176) ~[classes/:?]
2024-11-16T16:36:24,441 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:24,616 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-16T16:36:24,619 DEBUG [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/oldWALs
2024-11-16T16:36:24,619 INFO  [RS:0;303fe01965e8:36407 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C36407%2C1731774934631:(num 1731774935277)
2024-11-16T16:36:24,619 DEBUG [RS:0;303fe01965e8:36407 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:24,619 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:24,619 INFO  [RS:0;303fe01965e8:36407 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:36:24,620 INFO  [RS:0;303fe01965e8:36407 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T16:36:24,620 INFO  [RS:0;303fe01965e8:36407 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:36:24,620 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:36:24,620 INFO  [RS:0;303fe01965e8:36407 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36407
2024-11-16T16:36:24,631 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor116.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:24,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,36407,1731774934631
2024-11-16T16:36:24,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:36:24,684 INFO  [RS:0;303fe01965e8:36407 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:36:24,694 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,36407,1731774934631]
2024-11-16T16:36:24,705 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,36407,1731774934631 already deleted, retry=false
2024-11-16T16:36:24,705 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,36407,1731774934631 expired; onlineServers=0
2024-11-16T16:36:24,705 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,39189,1731774934458' *****
2024-11-16T16:36:24,705 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:36:24,705 INFO  [M:0;303fe01965e8:39189 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:36:24,705 INFO  [M:0;303fe01965e8:39189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:36:24,706 DEBUG [M:0;303fe01965e8:39189 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:36:24,706 DEBUG [M:0;303fe01965e8:39189 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:36:24,706 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:36:24,706 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774934993 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774934993,5,FailOnTimeoutGroup]
2024-11-16T16:36:24,706 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774934993 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774934993,5,FailOnTimeoutGroup]
2024-11-16T16:36:24,706 INFO  [M:0;303fe01965e8:39189 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:36:24,706 INFO  [M:0;303fe01965e8:39189 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:36:24,706 DEBUG [M:0;303fe01965e8:39189 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:36:24,706 INFO  [M:0;303fe01965e8:39189 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:36:24,707 INFO  [M:0;303fe01965e8:39189 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:36:24,707 INFO  [M:0;303fe01965e8:39189 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:36:24,707 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:36:24,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:36:24,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:24,716 DEBUG [M:0;303fe01965e8:39189 {}] zookeeper.ZKUtil(347): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:36:24,716 WARN  [M:0;303fe01965e8:39189 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:36:24,716 INFO  [M:0;303fe01965e8:39189 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/.lastflushedseqids
2024-11-16T16:36:24,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741904_1090 (size=130)
2024-11-16T16:36:24,725 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741904_1090 (size=130)
2024-11-16T16:36:24,726 INFO  [M:0;303fe01965e8:39189 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:36:24,726 INFO  [M:0;303fe01965e8:39189 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:36:24,726 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:36:24,726 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:24,726 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:24,726 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:36:24,726 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:24,726 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB
2024-11-16T16:36:24,746 DEBUG [M:0;303fe01965e8:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb2f09351584345b7cc14506bea3dc0 is 82, key is hbase:meta,,1/info:regioninfo/1731774935659/Put/seqid=0
2024-11-16T16:36:24,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741905_1091 (size=5672)
2024-11-16T16:36:24,755 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741905_1091 (size=5672)
2024-11-16T16:36:24,755 INFO  [M:0;303fe01965e8:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb2f09351584345b7cc14506bea3dc0
2024-11-16T16:36:24,780 DEBUG [M:0;303fe01965e8:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4f836c07892487582219630f1744b0a is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731774936742/Put/seqid=0
2024-11-16T16:36:24,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741906_1092 (size=6256)
2024-11-16T16:36:24,785 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741906_1092 (size=6256)
2024-11-16T16:36:24,786 INFO  [M:0;303fe01965e8:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4f836c07892487582219630f1744b0a
2024-11-16T16:36:24,792 INFO  [M:0;303fe01965e8:39189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e4f836c07892487582219630f1744b0a
2024-11-16T16:36:24,795 INFO  [RS:0;303fe01965e8:36407 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:36:24,795 INFO  [RS:0;303fe01965e8:36407 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,36407,1731774934631; zookeeper connection closed.
2024-11-16T16:36:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36407-0x101448069510001, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:24,795 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31e5bb49 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31e5bb49
2024-11-16T16:36:24,795 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-16T16:36:24,813 DEBUG [M:0;303fe01965e8:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8d6e23460dc4d5ab4bf0d91abe98452 is 69, key is 303fe01965e8,33539,1731774935781/rs:state/1731774935853/Put/seqid=0
2024-11-16T16:36:24,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741907_1093 (size=5224)
2024-11-16T16:36:24,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741907_1093 (size=5224)
2024-11-16T16:36:24,819 INFO  [M:0;303fe01965e8:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8d6e23460dc4d5ab4bf0d91abe98452
2024-11-16T16:36:24,842 DEBUG [M:0;303fe01965e8:39189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3db3eb4febd447cb8341601520dfb288 is 52, key is load_balancer_on/state:d/1731774935765/Put/seqid=0
2024-11-16T16:36:24,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741908_1094 (size=5056)
2024-11-16T16:36:24,847 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741908_1094 (size=5056)
2024-11-16T16:36:24,848 INFO  [M:0;303fe01965e8:39189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3db3eb4febd447cb8341601520dfb288
2024-11-16T16:36:24,854 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/abb2f09351584345b7cc14506bea3dc0 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/abb2f09351584345b7cc14506bea3dc0
2024-11-16T16:36:24,859 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/abb2f09351584345b7cc14506bea3dc0, entries=8, sequenceid=60, filesize=5.5 K
2024-11-16T16:36:24,860 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e4f836c07892487582219630f1744b0a as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e4f836c07892487582219630f1744b0a
2024-11-16T16:36:24,867 INFO  [M:0;303fe01965e8:39189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e4f836c07892487582219630f1744b0a
2024-11-16T16:36:24,867 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e4f836c07892487582219630f1744b0a, entries=6, sequenceid=60, filesize=6.1 K
2024-11-16T16:36:24,868 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e8d6e23460dc4d5ab4bf0d91abe98452 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e8d6e23460dc4d5ab4bf0d91abe98452
2024-11-16T16:36:24,875 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e8d6e23460dc4d5ab4bf0d91abe98452, entries=2, sequenceid=60, filesize=5.1 K
2024-11-16T16:36:24,876 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3db3eb4febd447cb8341601520dfb288 as hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3db3eb4febd447cb8341601520dfb288
2024-11-16T16:36:24,883 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3db3eb4febd447cb8341601520dfb288, entries=1, sequenceid=60, filesize=4.9 K
2024-11-16T16:36:24,884 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=60, compaction requested=false
2024-11-16T16:36:24,886 INFO  [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:24,886 DEBUG [M:0;303fe01965e8:39189 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774984726Disabling compacts and flushes for region at 1731774984726Disabling writes for close at 1731774984726Obtaining lock to block concurrent updates at 1731774984726Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731774984726Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1731774984727 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731774984728 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731774984728Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731774984746 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731774984746Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731774984762 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731774984779 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731774984779Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731774984792 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731774984813 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731774984813Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731774984825 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731774984841 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731774984841Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d162fb5: reopening flushed file at 1731774984853 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@786cf4: reopening flushed file at 1731774984859 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a70f670: reopening flushed file at 1731774984867 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dc374d6: reopening flushed file at 1731774984875 (+8 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 158ms, sequenceid=60, compaction requested=false at 1731774984884 (+9 ms)Writing region close event to WAL at 1731774984886 (+2 ms)Closed at 1731774984886
2024-11-16T16:36:24,887 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:24,887 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:24,887 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:24,887 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:24,887 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:24,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38241 is added to blk_1073741889_1072 (size=1045)
2024-11-16T16:36:24,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37865 is added to blk_1073741889_1072 (size=1045)
2024-11-16T16:36:24,890 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:36:24,890 INFO  [M:0;303fe01965e8:39189 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:36:24,890 INFO  [M:0;303fe01965e8:39189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39189
2024-11-16T16:36:24,891 INFO  [M:0;303fe01965e8:39189 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:36:25,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:25,003 INFO  [M:0;303fe01965e8:39189 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:36:25,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39189-0x101448069510000, quorum=127.0.0.1:54861, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:25,007 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2817e38a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:25,007 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f9cc2c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:25,007 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:25,007 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@278e888c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:25,008 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7cac94da{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:25,009 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:25,009 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f75f6fb {}] ipc.Client$Connection(956): Interrupted while trying for connection
2024-11-16T16:36:25,009 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5c3e11 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:41585,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:46543 , LocalHost:localPort 303fe01965e8/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-16T16:36:25,010 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:25,010 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:25,010 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid e28f5a31-d7b1-4830-abef-9e7c86b1febc) service to localhost/127.0.0.1:36911
2024-11-16T16:36:25,010 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f75f6fb {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41585,null,null])
java.net.ConnectException: Call From 303fe01965e8/172.17.0.2 to localhost:46543 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-16T16:36:25,010 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f75f6fb {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38241,null,null])
java.io.IOException: No block pool offer service for bpid=BP-2075037832-172.17.0.2-1731774932374
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:25,011 ERROR [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@5c3e11 {}] datanode.DataNode(1743): Cannot find BPOfferService for reporting block received for bpid=BP-2075037832-172.17.0.2-1731774932374
2024-11-16T16:36:25,011 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f75f6fb {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:41585,null,null], DatanodeInfoWithStorage[127.0.0.1:38241,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-2075037832-172.17.0.2-1731774932374:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:41585,null,null], DatanodeInfoWithStorage[127.0.0.1:38241,null,null]]
2024-11-16T16:36:25,011 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data3/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:25,012 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data4/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:25,012 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:25,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@592c6d7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:25,017 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1d39bcd7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:25,017 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:25,017 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70c175e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:25,018 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7645500{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:25,019 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:25,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:25,019 WARN  [BP-2075037832-172.17.0.2-1731774932374 heartbeating to localhost/127.0.0.1:36911 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2075037832-172.17.0.2-1731774932374 (Datanode Uuid 3b3f2ee1-04d9-4a6e-8ae9-cedb0cf6e289) service to localhost/127.0.0.1:36911
2024-11-16T16:36:25,019 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:25,020 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data7/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:25,020 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/cluster_a8bc844e-7237-22b5-3ab0-b14dcb31a95c/data/data8/current/BP-2075037832-172.17.0.2-1731774932374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:25,020 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:25,025 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2cd29190{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:36:25,026 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3cc2b5d2{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:25,026 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:25,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48202e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:25,026 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@57980111{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:25,036 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:36:25,077 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:36:25,086 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=157 (was 81)
Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36911 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36911
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43373
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36911 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:36911 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fad10bef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:36911
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36911
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fad10bef5c0.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:36911
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36911
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36911
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36911
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-16-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:43373
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-17-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=431 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=174 (was 140) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3796 (was 4720)
2024-11-16T16:36:25,094 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=157, OpenFileDescriptor=431, MaxFileDescriptor=1048576, SystemLoadAverage=174, ProcessCount=11, AvailableMemoryMB=3796
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.log.dir so I do NOT create it in target/test-data/2c2941a4-d976-7235-c028-d9707460360b
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/91d68f5a-df5c-4ffb-d24b-cb1d98a1454a/hadoop.tmp.dir so I do NOT create it in target/test-data/2c2941a4-d976-7235-c028-d9707460360b
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4, deleteOnExit=true
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:36:25,095 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/test.cache.data in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:36:25,096 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:36:25,096 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:36:25,097 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:36:25,098 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:36:25,112 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:36:25,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:25,487 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:25,491 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:25,493 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:25,493 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:25,493 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:36:25,493 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:25,494 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@115eaf6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:25,494 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@209ebed6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:25,604 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@585ec7cf{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-42219-hadoop-hdfs-3_4_1-tests_jar-_-any-9128009232361782584/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:36:25,605 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@27bc6094{HTTP/1.1, (http/1.1)}{localhost:42219}
2024-11-16T16:36:25,605 INFO  [Time-limited test {}] server.Server(415): Started @161380ms
2024-11-16T16:36:25,617 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:36:25,633 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:25,886 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:25,890 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:25,896 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:25,896 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:25,896 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:36:25,897 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9dc8204{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:25,897 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21e3a41b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:25,999 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@417b29aa{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-35007-hadoop-hdfs-3_4_1-tests_jar-_-any-5342271488836841221/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:26,000 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@350106ae{HTTP/1.1, (http/1.1)}{localhost:35007}
2024-11-16T16:36:26,000 INFO  [Time-limited test {}] server.Server(415): Started @161776ms
2024-11-16T16:36:26,002 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:26,035 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:26,039 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:26,039 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:26,040 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:26,040 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:36:26,040 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@53d4da47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:26,041 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7bb6ca3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:26,147 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@598d9c74{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-46819-hadoop-hdfs-3_4_1-tests_jar-_-any-9109321267885041810/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:26,147 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25c074d6{HTTP/1.1, (http/1.1)}{localhost:46819}
2024-11-16T16:36:26,147 INFO  [Time-limited test {}] server.Server(415): Started @161923ms
2024-11-16T16:36:26,148 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:26,443 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:26,633 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:26,691 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:36:26,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,712 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,713 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,717 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,717 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,717 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:26,721 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:27,254 WARN  [Thread-1219 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data2/current/BP-2119659671-172.17.0.2-1731774985124/current, will proceed with Du for space computation calculation, 
2024-11-16T16:36:27,254 WARN  [Thread-1218 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data1/current/BP-2119659671-172.17.0.2-1731774985124/current, will proceed with Du for space computation calculation, 
2024-11-16T16:36:27,275 WARN  [Thread-1181 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:27,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe669dc5026a3fde5 with lease ID 0x2795e5058e1d7135: Processing first storage report for DS-df86c6c4-1300-4aed-afd1-b0012be56f63 from datanode DatanodeRegistration(127.0.0.1:33515, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=32927, infoSecurePort=0, ipcPort=38297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124)
2024-11-16T16:36:27,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe669dc5026a3fde5 with lease ID 0x2795e5058e1d7135: from storage DS-df86c6c4-1300-4aed-afd1-b0012be56f63 node DatanodeRegistration(127.0.0.1:33515, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=32927, infoSecurePort=0, ipcPort=38297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:27,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe669dc5026a3fde5 with lease ID 0x2795e5058e1d7135: Processing first storage report for DS-77598bd6-336a-4f46-a76a-eae0b7e53186 from datanode DatanodeRegistration(127.0.0.1:33515, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=32927, infoSecurePort=0, ipcPort=38297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124)
2024-11-16T16:36:27,278 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe669dc5026a3fde5 with lease ID 0x2795e5058e1d7135: from storage DS-77598bd6-336a-4f46-a76a-eae0b7e53186 node DatanodeRegistration(127.0.0.1:33515, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=32927, infoSecurePort=0, ipcPort=38297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:27,444 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:27,519 WARN  [Thread-1229 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data3/current/BP-2119659671-172.17.0.2-1731774985124/current, will proceed with Du for space computation calculation, 
2024-11-16T16:36:27,519 WARN  [Thread-1230 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data4/current/BP-2119659671-172.17.0.2-1731774985124/current, will proceed with Du for space computation calculation, 
2024-11-16T16:36:27,538 WARN  [Thread-1204 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:27,540 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf88311dda4f14221 with lease ID 0x2795e5058e1d7136: Processing first storage report for DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b from datanode DatanodeRegistration(127.0.0.1:40583, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=42591, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124)
2024-11-16T16:36:27,541 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf88311dda4f14221 with lease ID 0x2795e5058e1d7136: from storage DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b node DatanodeRegistration(127.0.0.1:40583, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=42591, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:27,541 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf88311dda4f14221 with lease ID 0x2795e5058e1d7136: Processing first storage report for DS-ef310239-a626-499c-af24-1438aad48338 from datanode DatanodeRegistration(127.0.0.1:40583, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=42591, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124)
2024-11-16T16:36:27,541 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf88311dda4f14221 with lease ID 0x2795e5058e1d7136: from storage DS-ef310239-a626-499c-af24-1438aad48338 node DatanodeRegistration(127.0.0.1:40583, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=42591, infoSecurePort=0, ipcPort=40239, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:27,606 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b
2024-11-16T16:36:27,610 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/zookeeper_0, clientPort=62941, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:36:27,611 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62941
2024-11-16T16:36:27,612 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,613 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,622 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:36:27,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:36:27,624 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e with version=8
2024-11-16T16:36:27,624 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:36:27,627 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:36:27,628 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45601
2024-11-16T16:36:27,630 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45601 connecting to ZooKeeper ensemble=127.0.0.1:62941
2024-11-16T16:36:27,634 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:27,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:456010x0, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:36:27,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45601-0x101448139040000 connected
2024-11-16T16:36:27,786 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,788 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:27,791 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e, hbase.cluster.distributed=false
2024-11-16T16:36:27,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:36:27,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45601
2024-11-16T16:36:27,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45601
2024-11-16T16:36:27,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45601
2024-11-16T16:36:27,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45601
2024-11-16T16:36:27,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45601
2024-11-16T16:36:27,814 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:36:27,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,814 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:36:27,814 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:36:27,815 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:36:27,815 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:36:27,815 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:36:27,816 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46713
2024-11-16T16:36:27,817 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46713 connecting to ZooKeeper ensemble=127.0.0.1:62941
2024-11-16T16:36:27,818 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,820 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467130x0, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:36:27,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467130x0, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:27,834 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:36:27,840 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46713-0x101448139040001 connected
2024-11-16T16:36:27,841 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:36:27,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:36:27,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:36:27,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46713
2024-11-16T16:36:27,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46713
2024-11-16T16:36:27,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46713
2024-11-16T16:36:27,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46713
2024-11-16T16:36:27,844 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46713
2024-11-16T16:36:27,860 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:45601
2024-11-16T16:36:27,861 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,45601,1731774987626
2024-11-16T16:36:27,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:36:27,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:36:27,870 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,45601,1731774987626
2024-11-16T16:36:27,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:36:27,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:27,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:27,880 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:36:27,881 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,45601,1731774987626 from backup master directory
2024-11-16T16:36:27,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,45601,1731774987626
2024-11-16T16:36:27,890 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:36:27,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:36:27,890 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,45601,1731774987626
2024-11-16T16:36:27,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:36:27,895 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/hbase.id] with ID: 43c76080-2d87-4ac6-a722-95c945f532cd
2024-11-16T16:36:27,895 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/.tmp/hbase.id
2024-11-16T16:36:27,901 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:36:27,902 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:36:27,902 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/.tmp/hbase.id]:[hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/hbase.id]
2024-11-16T16:36:27,915 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:27,916 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:36:27,917 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-16T16:36:27,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:27,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:27,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:36:27,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:36:27,936 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:36:27,937 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:36:27,937 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:36:27,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:36:27,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:36:27,946 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store
2024-11-16T16:36:27,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:36:27,961 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:36:27,962 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:27,962 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:27,962 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731774987962Disabling compacts and flushes for region at 1731774987962Disabling writes for close at 1731774987962Writing region close event to WAL at 1731774987962Closed at 1731774987962
2024-11-16T16:36:27,963 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/.initializing
2024-11-16T16:36:27,963 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626
2024-11-16T16:36:27,966 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C45601%2C1731774987626, suffix=, logDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626, archiveDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/oldWALs, maxLogs=10
2024-11-16T16:36:27,966 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C45601%2C1731774987626.1731774987966
2024-11-16T16:36:27,973 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966
2024-11-16T16:36:27,974 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42591:42591),(127.0.0.1/127.0.0.1:32927:32927)]
2024-11-16T16:36:27,975 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:36:27,975 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:27,975 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,976 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,980 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:36:27,982 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:36:27,982 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:27,983 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:27,983 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:36:27,985 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:36:27,985 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:27,985 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:36:27,986 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:36:27,987 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:36:27,987 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:27,988 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:36:27,988 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:36:27,989 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:36:27,989 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:27,990 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:36:27,990 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,991 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,991 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,992 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,993 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,993 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:36:27,994 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:36:27,996 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:36:27,997 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=855341, jitterRate=0.08762243390083313}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:36:27,998 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731774987976Initializing all the Stores at 1731774987977 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774987977Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774987980 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774987980Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774987980Cleaning up temporary data from old regions at 1731774987993 (+13 ms)Region opened successfully at 1731774987997 (+4 ms)
2024-11-16T16:36:27,998 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:36:28,001 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471cd77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:36:28,002 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:36:28,002 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:36:28,002 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:36:28,002 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:36:28,003 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:36:28,003 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:36:28,004 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:36:28,006 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:36:28,007 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:36:28,017 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:36:28,017 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:36:28,018 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:36:28,027 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:36:28,028 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:36:28,029 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:36:28,038 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:36:28,039 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:36:28,048 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:36:28,051 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:36:28,059 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:36:28,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:28,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:28,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,070 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,45601,1731774987626, sessionid=0x101448139040000, setting cluster-up flag (Was=false)
2024-11-16T16:36:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,122 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:36:28,123 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,45601,1731774987626
2024-11-16T16:36:28,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,175 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:36:28,176 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,45601,1731774987626
2024-11-16T16:36:28,177 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:36:28,179 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:36:28,179 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:36:28,180 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:36:28,180 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,45601,1731774987626
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:36:28,181 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:36:28,182 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,183 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731775018183
2024-11-16T16:36:28,183 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:36:28,183 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:36:28,184 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:36:28,185 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:36:28,185 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:36:28,185 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:36:28,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:36:28,185 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:36:28,185 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774988185,5,FailOnTimeoutGroup]
2024-11-16T16:36:28,186 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,187 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:36:28,188 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774988185,5,FailOnTimeoutGroup]
2024-11-16T16:36:28,188 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,188 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:36:28,188 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,188 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:36:28,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:36:28,203 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:36:28,203 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e
2024-11-16T16:36:28,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:36:28,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:36:28,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:28,216 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:36:28,218 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:36:28,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,218 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,219 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:36:28,220 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:36:28,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,221 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,221 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:36:28,222 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:36:28,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,223 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,223 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:36:28,224 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:36:28,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,225 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:36:28,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740
2024-11-16T16:36:28,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740
2024-11-16T16:36:28,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:36:28,227 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:36:28,228 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:36:28,229 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:36:28,231 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:36:28,231 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=690663, jitterRate=-0.12177665531635284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:36:28,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731774988215Initializing all the Stores at 1731774988216 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988216Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988216Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774988216Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988216Cleaning up temporary data from old regions at 1731774988227 (+11 ms)Region opened successfully at 1731774988232 (+5 ms)
2024-11-16T16:36:28,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:36:28,232 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:36:28,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:36:28,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:36:28,232 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:36:28,233 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:36:28,233 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731774988232Disabling compacts and flushes for region at 1731774988232Disabling writes for close at 1731774988232Writing region close event to WAL at 1731774988233 (+1 ms)Closed at 1731774988233
2024-11-16T16:36:28,234 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:36:28,234 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:36:28,234 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:36:28,236 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:36:28,237 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:36:28,250 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(746): ClusterId : 43c76080-2d87-4ac6-a722-95c945f532cd
2024-11-16T16:36:28,250 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:36:28,260 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:36:28,260 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:36:28,270 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:36:28,271 DEBUG [RS:0;303fe01965e8:46713 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b49dcf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:36:28,284 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:46713
2024-11-16T16:36:28,284 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:36:28,284 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:36:28,284 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:36:28,285 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,45601,1731774987626 with port=46713, startcode=1731774987814
2024-11-16T16:36:28,285 DEBUG [RS:0;303fe01965e8:46713 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:36:28,288 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:36:28,288 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45601 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,289 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45601 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,290 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e
2024-11-16T16:36:28,291 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44217
2024-11-16T16:36:28,291 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:36:28,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:36:28,301 DEBUG [RS:0;303fe01965e8:46713 {}] zookeeper.ZKUtil(111): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,301 WARN  [RS:0;303fe01965e8:46713 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:36:28,302 INFO  [RS:0;303fe01965e8:46713 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:36:28,302 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,302 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,46713,1731774987814]
2024-11-16T16:36:28,306 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:36:28,307 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:36:28,308 INFO  [RS:0;303fe01965e8:46713 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:36:28,308 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,308 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:36:28,309 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:36:28,309 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,310 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:36:28,311 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:36:28,311 DEBUG [RS:0;303fe01965e8:46713 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,312 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,46713,1731774987814-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:36:28,335 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:36:28,335 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,46713,1731774987814-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,335 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,335 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.Replication(171): 303fe01965e8,46713,1731774987814 started
2024-11-16T16:36:28,349 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,349 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,46713,1731774987814, RpcServer on 303fe01965e8/172.17.0.2:46713, sessionid=0x101448139040001
2024-11-16T16:36:28,349 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:36:28,349 DEBUG [RS:0;303fe01965e8:46713 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,349 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,46713,1731774987814'
2024-11-16T16:36:28,349 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,46713,1731774987814'
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:36:28,350 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:36:28,351 DEBUG [RS:0;303fe01965e8:46713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:36:28,351 INFO  [RS:0;303fe01965e8:46713 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:36:28,351 INFO  [RS:0;303fe01965e8:46713 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:36:28,387 WARN  [303fe01965e8:45601 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:36:28,445 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:28,454 INFO  [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C46713%2C1731774987814, suffix=, logDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814, archiveDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs, maxLogs=32
2024-11-16T16:36:28,455 INFO  [RS:0;303fe01965e8:46713 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:28,466 INFO  [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:28,467 DEBUG [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42591:42591),(127.0.0.1/127.0.0.1:32927:32927)]
2024-11-16T16:36:28,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:28,638 DEBUG [303fe01965e8:45601 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:36:28,638 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,640 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,46713,1731774987814, state=OPENING
2024-11-16T16:36:28,690 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:36:28,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:28,702 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:36:28,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:36:28,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:36:28,702 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,46713,1731774987814}]
2024-11-16T16:36:28,856 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:36:28,858 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40311, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:36:28,862 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:36:28,862 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:36:28,863 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C46713%2C1731774987814.meta, suffix=.meta, logDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814, archiveDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs, maxLogs=32
2024-11-16T16:36:28,864 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta
2024-11-16T16:36:28,871 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta
2024-11-16T16:36:28,872 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32927:32927),(127.0.0.1/127.0.0.1:42591:42591)]
2024-11-16T16:36:28,873 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:36:28,874 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:36:28,874 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:36:28,876 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:36:28,877 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:36:28,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,878 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,878 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:36:28,879 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:36:28,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,880 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:36:28,881 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:36:28,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,886 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,886 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:36:28,887 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:36:28,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:28,888 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:36:28,888 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:36:28,889 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740
2024-11-16T16:36:28,890 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740
2024-11-16T16:36:28,892 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:36:28,892 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:36:28,892 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:36:28,894 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:36:28,895 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=848669, jitterRate=0.07913951575756073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:36:28,895 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:36:28,895 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731774988874Writing region info on filesystem at 1731774988874Initializing all the Stores at 1731774988875 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988875Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988876 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774988876Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731774988876Cleaning up temporary data from old regions at 1731774988892 (+16 ms)Running coprocessor post-open hooks at 1731774988895 (+3 ms)Region opened successfully at 1731774988895
2024-11-16T16:36:28,897 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731774988855
2024-11-16T16:36:28,900 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:36:28,900 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:36:28,901 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,902 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,46713,1731774987814, state=OPEN
2024-11-16T16:36:28,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:36:28,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:36:28,941 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,46713,1731774987814
2024-11-16T16:36:28,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:36:28,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:36:28,944 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:36:28,944 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,46713,1731774987814 in 239 msec
2024-11-16T16:36:28,947 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:36:28,948 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 710 msec
2024-11-16T16:36:28,949 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:36:28,949 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:36:28,951 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:36:28,951 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,46713,1731774987814, seqNum=-1]
2024-11-16T16:36:28,951 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:36:28,953 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49925, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:36:28,966 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 780 msec
2024-11-16T16:36:28,966 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731774988966, completionTime=-1
2024-11-16T16:36:28,966 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:36:28,966 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731775048969
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775108969
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,969 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,970 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:45601, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,970 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,970 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:28,972 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:36:28,974 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.083sec
2024-11-16T16:36:28,974 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:36:28,974 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:36:28,974 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:36:28,975 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:36:28,975 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:36:28,975 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:36:28,975 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:36:28,978 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:36:28,978 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:36:28,978 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45601,1731774987626-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:36:29,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641c7c70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:36:29,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45601,-1 for getting cluster id
2024-11-16T16:36:29,051 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:36:29,053 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '43c76080-2d87-4ac6-a722-95c945f532cd'
2024-11-16T16:36:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:36:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "43c76080-2d87-4ac6-a722-95c945f532cd"

2024-11-16T16:36:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3455837c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:36:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45601,-1]
2024-11-16T16:36:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:36:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:29,056 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:36:29,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c17f84d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:36:29,058 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:36:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,46713,1731774987814, seqNum=-1]
2024-11-16T16:36:29,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:36:29,061 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:36:29,063 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,45601,1731774987626
2024-11-16T16:36:29,064 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:36:29,067 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:36:29,067 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-16T16:36:29,067 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-16T16:36:29,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T16:36:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45601,1731774987626
2024-11-16T16:36:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f5db474
2024-11-16T16:36:29,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T16:36:29,071 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T16:36:29,072 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-16T16:36:29,072 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-16T16:36:29,072 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:36:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-16T16:36:29,075 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T16:36:29,075 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:29,075 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-16T16:36:29,076 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T16:36:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:36:29,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741835_1011 (size=395)
2024-11-16T16:36:29,084 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741835_1011 (size=395)
2024-11-16T16:36:29,086 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f1e14f21609a5d55957d236c91429d5d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e
2024-11-16T16:36:29,093 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33515 is added to blk_1073741836_1012 (size=78)
2024-11-16T16:36:29,094 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40583 is added to blk_1073741836_1012 (size=78)
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing f1e14f21609a5d55957d236c91429d5d, disabling compactions & flushes
2024-11-16T16:36:29,094 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d. after waiting 0 ms
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,094 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,094 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for f1e14f21609a5d55957d236c91429d5d:
Waiting for close lock at 1731774989094Disabling compacts and flushes for region at 1731774989094Disabling writes for close at 1731774989094Writing region close event to WAL at 1731774989094Closed at 1731774989094
2024-11-16T16:36:29,096 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T16:36:29,096 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731774989096"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731774989096"}]},"ts":"1731774989096"}
2024-11-16T16:36:29,099 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T16:36:29,101 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T16:36:29,101 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774989101"}]},"ts":"1731774989101"}
2024-11-16T16:36:29,103 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-16T16:36:29,104 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f1e14f21609a5d55957d236c91429d5d, ASSIGN}]
2024-11-16T16:36:29,105 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f1e14f21609a5d55957d236c91429d5d, ASSIGN
2024-11-16T16:36:29,106 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f1e14f21609a5d55957d236c91429d5d, ASSIGN; state=OFFLINE, location=303fe01965e8,46713,1731774987814; forceNewPlan=false, retain=false
2024-11-16T16:36:29,257 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1e14f21609a5d55957d236c91429d5d, regionState=OPENING, regionLocation=303fe01965e8,46713,1731774987814
2024-11-16T16:36:29,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f1e14f21609a5d55957d236c91429d5d, ASSIGN because future has completed
2024-11-16T16:36:29,260 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1e14f21609a5d55957d236c91429d5d, server=303fe01965e8,46713,1731774987814}]
2024-11-16T16:36:29,417 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,418 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f1e14f21609a5d55957d236c91429d5d, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:36:29,418 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,418 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:36:29,418 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,418 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,419 INFO  [StoreOpener-f1e14f21609a5d55957d236c91429d5d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f1e14f21609a5d55957d236c91429d5d 
2024-11-16T16:36:29,421 INFO  [StoreOpener-f1e14f21609a5d55957d236c91429d5d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1e14f21609a5d55957d236c91429d5d columnFamilyName info
2024-11-16T16:36:29,421 DEBUG [StoreOpener-f1e14f21609a5d55957d236c91429d5d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:36:29,422 INFO  [StoreOpener-f1e14f21609a5d55957d236c91429d5d-1 {}] regionserver.HStore(327): Store=f1e14f21609a5d55957d236c91429d5d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:36:29,422 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,422 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,423 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,423 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,423 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,425 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,429 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:36:29,430 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f1e14f21609a5d55957d236c91429d5d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=764093, jitterRate=-0.028405681252479553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:36:29,430 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:29,431 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f1e14f21609a5d55957d236c91429d5d:
Running coprocessor pre-open hook at 1731774989418Writing region info on filesystem at 1731774989418Initializing all the Stores at 1731774989419 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731774989419Cleaning up temporary data from old regions at 1731774989423 (+4 ms)Running coprocessor post-open hooks at 1731774989430 (+7 ms)Region opened successfully at 1731774989431 (+1 ms)
2024-11-16T16:36:29,432 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d., pid=6, masterSystemTime=1731774989413
2024-11-16T16:36:29,435 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,435 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:29,436 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f1e14f21609a5d55957d236c91429d5d, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,46713,1731774987814
2024-11-16T16:36:29,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f1e14f21609a5d55957d236c91429d5d, server=303fe01965e8,46713,1731774987814 because future has completed
2024-11-16T16:36:29,443 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T16:36:29,443 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f1e14f21609a5d55957d236c91429d5d, server=303fe01965e8,46713,1731774987814 in 180 msec
2024-11-16T16:36:29,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:29,447 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T16:36:29,447 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f1e14f21609a5d55957d236c91429d5d, ASSIGN in 339 msec
2024-11-16T16:36:29,448 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T16:36:29,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731774989448"}]},"ts":"1731774989448"}
2024-11-16T16:36:29,450 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-16T16:36:29,452 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T16:36:29,454 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 380 msec
2024-11-16T16:36:29,635 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:30,446 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:30,636 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:31,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:36:31,189 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T16:36:31,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-16T16:36:31,190 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-16T16:36:31,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:36:31,191 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T16:36:31,447 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:31,637 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:32,448 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:32,638 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:33,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:33,638 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:34,377 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:36:34,400 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,400 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,400 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,401 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,402 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,406 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,407 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,410 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:34,417 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:36:34,419 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-16T16:36:34,449 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:34,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:35,450 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:35,639 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:36,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:36,640 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:37,451 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:37,642 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:38,452 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:38,642 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:36:39,137 INFO  [RPCClient-NioEventLoopGroup-4-16 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-16T16:36:39,137 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-16T16:36:39,141 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-16T16:36:39,142 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:39,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d., hostname=303fe01965e8,46713,1731774987814, seqNum=2]
2024-11-16T16:36:39,453 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:39,643 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:40,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:40,644 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:41,149 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:41,150 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,150 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,150 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK], DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]) is bad.
2024-11-16T16:36:41,150 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK], DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]) is bad.
2024-11-16T16:36:41,150 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,150 WARN  [PacketResponder: BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40583] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,151 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK], DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40583,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]) is bad.
2024-11-16T16:36:41,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:46584 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46584 dst: /127.0.0.1:33515
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1856240338_22 at /127.0.0.1:39080 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39080 dst: /127.0.0.1:40583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:39112 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39112 dst: /127.0.0.1:40583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,151 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1856240338_22 at /127.0.0.1:46552 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46552 dst: /127.0.0.1:33515
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:39106 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40583:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39106 dst: /127.0.0.1:40583
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,152 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:46568 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:46568 dst: /127.0.0.1:33515
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,155 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@598d9c74{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:41,155 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@25c074d6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:41,156 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:41,156 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7bb6ca3f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:41,156 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@53d4da47{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:41,158 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:41,158 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:41,158 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 1f28b283-37b1-4694-903c-2a183702a7cd) service to localhost/127.0.0.1:44217
2024-11-16T16:36:41,158 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:41,158 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data3/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:41,158 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data4/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:41,159 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:41,166 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:41,170 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:41,176 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:41,176 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:41,176 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:36:41,184 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6143724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:41,184 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61be23b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:41,287 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dfcaa19{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-32977-hadoop-hdfs-3_4_1-tests_jar-_-any-13496935713822158864/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:41,288 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3e7340de{HTTP/1.1, (http/1.1)}{localhost:32977}
2024-11-16T16:36:41,288 INFO  [Time-limited test {}] server.Server(415): Started @177063ms
2024-11-16T16:36:41,289 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:41,305 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,305 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,305 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:41,306 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:59190 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59190 dst: /127.0.0.1:33515
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,306 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:59176 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59176 dst: /127.0.0.1:33515
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,306 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1856240338_22 at /127.0.0.1:59196 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33515:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59196 dst: /127.0.0.1:33515
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:41,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@417b29aa{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:41,338 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@350106ae{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:41,338 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:41,338 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21e3a41b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:41,339 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9dc8204{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:41,340 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:41,340 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:41,340 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:41,340 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 6f116c78-038d-46f0-946d-fce3c8fe505a) service to localhost/127.0.0.1:44217
2024-11-16T16:36:41,340 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data1/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:41,340 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data2/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:41,340 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:41,348 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:41,352 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:41,353 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:41,353 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:41,353 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:36:41,354 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1173a7e2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:41,354 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36f6f2a9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:41,454 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:41,475 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6023d9e0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-40125-hadoop-hdfs-3_4_1-tests_jar-_-any-16497668392882883640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:41,476 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6432dc3{HTTP/1.1, (http/1.1)}{localhost:40125}
2024-11-16T16:36:41,476 INFO  [Time-limited test {}] server.Server(415): Started @177251ms
2024-11-16T16:36:41,477 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:41,645 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:41,735 WARN  [Thread-1353 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:41,738 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8edfb8db15d31205 with lease ID 0x2795e5058e1d7137: from storage DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b node DatanodeRegistration(127.0.0.1:40785, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=43393, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:36:41,738 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8edfb8db15d31205 with lease ID 0x2795e5058e1d7137: from storage DS-ef310239-a626-499c-af24-1438aad48338 node DatanodeRegistration(127.0.0.1:40785, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=43393, infoSecurePort=0, ipcPort=33165, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:41,926 WARN  [Thread-1373 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:41,928 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd7fa71afa5539d1 with lease ID 0x2795e5058e1d7138: from storage DS-df86c6c4-1300-4aed-afd1-b0012be56f63 node DatanodeRegistration(127.0.0.1:43039, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=44049, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:41,929 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd7fa71afa5539d1 with lease ID 0x2795e5058e1d7138: from storage DS-77598bd6-336a-4f46-a76a-eae0b7e53186 node DatanodeRegistration(127.0.0.1:43039, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=44049, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:42,455 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:42,510 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-16T16:36:42,513 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-16T16:36:42,515 ERROR [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:42,515 WARN  [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:42,515 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C46713%2C1731774987814:(num 1731774988455) roll requested
2024-11-16T16:36:42,516 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:42,523 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 newFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:42,523 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:42,523 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:42,523 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:42,523 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:42,523 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:42,524 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:42,524 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:42,524 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:42,524 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:42,525 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44049:44049),(127.0.0.1/127.0.0.1:43393:43393)]
2024-11-16T16:36:42,525 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 is not closed yet, will try archiving it next time
2024-11-16T16:36:42,525 WARN  [IPC Server handler 1 on default port 44217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-11-16T16:36:42,525 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 after 1ms
2024-11-16T16:36:42,646 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:43,456 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:43,647 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:44,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:44,528 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-16T16:36:44,648 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:44,738 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-16T16:36:45,457 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:45,648 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:46,458 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:46,526 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 after 4002ms
2024-11-16T16:36:46,532 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:46,532 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43039,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK], DatanodeInfoWithStorage[127.0.0.1:40785,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:43039,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]) is bad.
2024-11-16T16:36:46,533 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:58542 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:43039:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:58542 dst: /127.0.0.1:43039
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:46,533 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:59324 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:40785:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59324 dst: /127.0.0.1:40785
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:46,580 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6023d9e0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:46,581 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6432dc3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:46,581 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:46,581 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36f6f2a9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:46,581 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1173a7e2{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:46,582 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:46,582 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:46,582 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 6f116c78-038d-46f0-946d-fce3c8fe505a) service to localhost/127.0.0.1:44217
2024-11-16T16:36:46,582 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:46,583 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data1/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:46,584 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data2/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:46,584 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:46,592 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:46,596 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:46,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:46,597 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:46,597 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:36:46,598 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48f4fa87{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:46,598 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@686dd8bf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:46,649 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:46,704 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10e0b03{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-36579-hadoop-hdfs-3_4_1-tests_jar-_-any-15997339348021952283/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:46,705 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3c894b0d{HTTP/1.1, (http/1.1)}{localhost:36579}
2024-11-16T16:36:46,705 INFO  [Time-limited test {}] server.Server(415): Started @182480ms
2024-11-16T16:36:46,707 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:46,738 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:46,738 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-9093086_22 at /127.0.0.1:59340 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:40785:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59340 dst: /127.0.0.1:40785
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:46,740 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dfcaa19{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:46,740 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3e7340de{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:36:46,740 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:36:46,740 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61be23b1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:36:46,740 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6143724{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:36:46,741 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:36:46,741 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:36:46,742 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:36:46,742 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 1f28b283-37b1-4694-903c-2a183702a7cd) service to localhost/127.0.0.1:44217
2024-11-16T16:36:46,742 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data3/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:46,742 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data4/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:36:46,743 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:36:46,763 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:36:46,766 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:36:46,769 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:36:46,769 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:36:46,769 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:36:46,770 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a0c1f66{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:36:46,770 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f2d69a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:36:46,875 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@478c28d0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/java.io.tmpdir/jetty-localhost-43643-hadoop-hdfs-3_4_1-tests_jar-_-any-11084576677387760057/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:36:46,876 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8cbc01a{HTTP/1.1, (http/1.1)}{localhost:43643}
2024-11-16T16:36:46,876 INFO  [Time-limited test {}] server.Server(415): Started @182651ms
2024-11-16T16:36:46,878 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:36:47,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:47,529 WARN  [Thread-1427 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:47,531 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43ff4c0f1b8149d1 with lease ID 0x2795e5058e1d7139: from storage DS-df86c6c4-1300-4aed-afd1-b0012be56f63 node DatanodeRegistration(127.0.0.1:39091, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=39815, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:47,531 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43ff4c0f1b8149d1 with lease ID 0x2795e5058e1d7139: from storage DS-77598bd6-336a-4f46-a76a-eae0b7e53186 node DatanodeRegistration(127.0.0.1:39091, datanodeUuid=6f116c78-038d-46f0-946d-fce3c8fe505a, infoPort=39815, infoSecurePort=0, ipcPort=43297, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:47,650 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:47,798 WARN  [Thread-1447 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:36:47,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x436729e48f16bbac with lease ID 0x2795e5058e1d713a: from storage DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b node DatanodeRegistration(127.0.0.1:44855, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=34811, infoSecurePort=0, ipcPort=33329, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:36:47,800 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x436729e48f16bbac with lease ID 0x2795e5058e1d713a: from storage DS-ef310239-a626-499c-af24-1438aad48338 node DatanodeRegistration(127.0.0.1:44855, datanodeUuid=1f28b283-37b1-4694-903c-2a183702a7cd, infoPort=34811, infoSecurePort=0, ipcPort=33329, storageInfo=lv=-57;cid=testClusterID;nsid=527074264;c=1731774985124), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:36:47,906 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-16T16:36:47,909 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-16T16:36:47,912 ERROR [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40785,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:47,912 WARN  [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40785,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:47,912 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C46713%2C1731774987814:(num 1731775002516) roll requested
2024-11-16T16:36:47,912 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:47,919 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 newFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:47,920 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:47,920 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:47,920 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:47,920 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:47,920 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:47,920 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:47,920 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40785,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:47,920 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40785,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:47,920 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:47,921 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39815:39815),(127.0.0.1/127.0.0.1:34811:34811)]
2024-11-16T16:36:47,921 WARN  [IPC Server handler 0 on default port 44217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-16T16:36:47,921 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 is not closed yet, will try archiving it next time
2024-11-16T16:36:47,921 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 after 1ms
2024-11-16T16:36:48,459 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:48,651 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:49,460 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:49,651 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:49,922 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:49,928 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912 newFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:49,928 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:49,928 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:49,928 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:49,928 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:49,928 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:49,929 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:49,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741838_1019 (size=1264)
2024-11-16T16:36:49,931 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741838_1019 (size=1264)
2024-11-16T16:36:49,931 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 is not closed yet, will try archiving it next time
2024-11-16T16:36:49,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39815:39815),(127.0.0.1/127.0.0.1:34811:34811)]
2024-11-16T16:36:49,940 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 is not closed yet, will try archiving it next time
2024-11-16T16:36:49,941 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:49,941 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:49,941 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455 after 0ms
2024-11-16T16:36:49,941 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:49,950 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731774989431/Put/vlen=218/seqid=0]
2024-11-16T16:36:49,951 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731774999147/Put/vlen=1045/seqid=0]
2024-11-16T16:36:49,951 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731774988455
2024-11-16T16:36:49,951 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:49,951 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:49,951 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 after 0ms
2024-11-16T16:36:49,951 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:49,956 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731775002514/Put/vlen=1045/seqid=0]
2024-11-16T16:36:49,956 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731775004529/Put/vlen=1045/seqid=0]
2024-11-16T16:36:49,956 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516
2024-11-16T16:36:49,956 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:49,956 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:49,957 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912 after 1ms
2024-11-16T16:36:49,957 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775007912
2024-11-16T16:36:49,960 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731775007911/Put/vlen=1045/seqid=0]
2024-11-16T16:36:49,960 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:49,960 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:49,961 WARN  [IPC Server handler 1 on default port 44217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-16T16:36:49,961 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 after 1ms
2024-11-16T16:36:50,461 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:50,652 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:50,803 WARN  [ResponseProcessor for block BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:50,803 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1856240338_22 at /127.0.0.1:48048 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44855:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:48048 dst: /127.0.0.1:44855
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:50,803 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1856240338_22 at /127.0.0.1:59100 [Receiving block BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39091:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:59100 dst: /127.0.0.1:39091
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39091 remote=/127.0.0.1:59100]. Total timeout mills is 60000, 59125 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:36:50,803 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 block BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39091,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK], DatanodeInfoWithStorage[127.0.0.1:44855,DS-6ec9a388-2e2b-4955-b0c7-ee4e2f94490b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39091,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]) is bad.
2024-11-16T16:36:50,804 WARN  [DataStreamer for file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 block BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:50,808 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741839_1022 (size=85)
2024-11-16T16:36:51,462 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:51,653 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:51,922 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775002516 after 4002ms
2024-11-16T16:36:52,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:52,654 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:53,463 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:53,533 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-16T16:36:53,654 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:53,962 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 after 4002ms
2024-11-16T16:36:53,962 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:53,966 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:53,966 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-16T16:36:53,967 ERROR [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:53,967 WARN  [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:53,967 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C46713%2C1731774987814.meta:.meta(num 1731774988864) roll requested
2024-11-16T16:36:53,967 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.meta.1731775013967.meta
2024-11-16T16:36:53,974 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:53,974 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:53,974 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:53,975 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:53,975 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:53,975 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731775013967.meta
2024-11-16T16:36:53,975 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:53,975 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:53,976 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta
2024-11-16T16:36:53,976 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39815:39815),(127.0.0.1/127.0.0.1:34811:34811)]
2024-11-16T16:36:53,976 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta is not closed yet, will try archiving it next time
2024-11-16T16:36:53,976 WARN  [IPC Server handler 0 on default port 44217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014
2024-11-16T16:36:53,976 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta after 0ms
2024-11-16T16:36:53,994 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/info/a687e9ab159647a6bdcd9954c40967c3 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d./info:regioninfo/1731774989435/Put/seqid=0
2024-11-16T16:36:53,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741841_1025 (size=7125)
2024-11-16T16:36:53,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741841_1025 (size=7125)
2024-11-16T16:36:54,000 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/info/a687e9ab159647a6bdcd9954c40967c3
2024-11-16T16:36:54,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/ns/aed6ad2879a7436a8b34e66865dff144 is 43, key is default/ns:d/1731774988954/Put/seqid=0
2024-11-16T16:36:54,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741842_1026 (size=5153)
2024-11-16T16:36:54,030 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741842_1026 (size=5153)
2024-11-16T16:36:54,031 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/ns/aed6ad2879a7436a8b34e66865dff144
2024-11-16T16:36:54,051 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/table/a67c9d2ba99a4780a067d52672498ddb is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731774989448/Put/seqid=0
2024-11-16T16:36:54,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741843_1027 (size=5438)
2024-11-16T16:36:54,057 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741843_1027 (size=5438)
2024-11-16T16:36:54,057 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/table/a67c9d2ba99a4780a067d52672498ddb
2024-11-16T16:36:54,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/info/a687e9ab159647a6bdcd9954c40967c3 as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/info/a687e9ab159647a6bdcd9954c40967c3
2024-11-16T16:36:54,070 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/info/a687e9ab159647a6bdcd9954c40967c3, entries=10, sequenceid=11, filesize=7.0 K
2024-11-16T16:36:54,071 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/ns/aed6ad2879a7436a8b34e66865dff144 as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/ns/aed6ad2879a7436a8b34e66865dff144
2024-11-16T16:36:54,077 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/ns/aed6ad2879a7436a8b34e66865dff144, entries=2, sequenceid=11, filesize=5.0 K
2024-11-16T16:36:54,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/.tmp/table/a67c9d2ba99a4780a067d52672498ddb as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/table/a67c9d2ba99a4780a067d52672498ddb
2024-11-16T16:36:54,086 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/table/a67c9d2ba99a4780a067d52672498ddb, entries=2, sequenceid=11, filesize=5.3 K
2024-11-16T16:36:54,087 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 121ms, sequenceid=11, compaction requested=false
2024-11-16T16:36:54,088 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-16T16:36:54,088 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing f1e14f21609a5d55957d236c91429d5d 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-16T16:36:54,088 ERROR [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:54,089 WARN  [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e-prefix:303fe01965e8,46713,1731774987814 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:54,089 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C46713%2C1731774987814:(num 1731775009922) roll requested
2024-11-16T16:36:54,089 INFO  [regionserver/303fe01965e8:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C46713%2C1731774987814.1731775014089
2024-11-16T16:36:54,095 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 newFile=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775014089
2024-11-16T16:36:54,095 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,095 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,095 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,095 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,095 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,095 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775014089
2024-11-16T16:36:54,096 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:54,096 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-2119659671-172.17.0.2-1731774985124:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy43.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy44.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor114.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:54,096 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:54,097 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 after 1ms
2024-11-16T16:36:54,097 DEBUG [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39815:39815),(127.0.0.1/127.0.0.1:34811:34811)]
2024-11-16T16:36:54,097 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.1731775009922 to hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs/303fe01965e8%2C46713%2C1731774987814.1731775009922
2024-11-16T16:36:54,114 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/.tmp/info/f56ea5cb994147608ddc6f9d504a0e57 is 1080, key is row1002/info:/1731774999147/Put/seqid=0
2024-11-16T16:36:54,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741845_1029 (size=9270)
2024-11-16T16:36:54,119 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741845_1029 (size=9270)
2024-11-16T16:36:54,120 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/.tmp/info/f56ea5cb994147608ddc6f9d504a0e57
2024-11-16T16:36:54,127 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/.tmp/info/f56ea5cb994147608ddc6f9d504a0e57 as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/info/f56ea5cb994147608ddc6f9d504a0e57
2024-11-16T16:36:54,134 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/info/f56ea5cb994147608ddc6f9d504a0e57, entries=4, sequenceid=8, filesize=9.1 K
2024-11-16T16:36:54,135 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for f1e14f21609a5d55957d236c91429d5d in 47ms, sequenceid=8, compaction requested=false
2024-11-16T16:36:54,135 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f1e14f21609a5d55957d236c91429d5d:

2024-11-16T16:36:54,141 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:36:54,141 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:36:54,141 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:54,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:54,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:54,141 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:36:54,141 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:36:54,141 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=154921661, stopped=false
2024-11-16T16:36:54,142 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,45601,1731774987626
2024-11-16T16:36:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:36:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:54,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:54,173 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:36:54,174 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:36:54,174 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:54,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:54,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:54,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:36:54,174 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,46713,1731774987814' *****
2024-11-16T16:36:54,175 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:36:54,175 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(3091): Received CLOSE for f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,46713,1731774987814
2024-11-16T16:36:54,175 INFO  [RS:0;303fe01965e8:46713 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:46713.
2024-11-16T16:36:54,176 DEBUG [RS:0;303fe01965e8:46713 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f1e14f21609a5d55957d236c91429d5d, disabling compactions & flushes
2024-11-16T16:36:54,176 DEBUG [RS:0;303fe01965e8:46713 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:54,176 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d. after waiting 0 ms
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:36:54,176 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-16T16:36:54,176 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, f1e14f21609a5d55957d236c91429d5d=TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.}
2024-11-16T16:36:54,176 DEBUG [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f1e14f21609a5d55957d236c91429d5d
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:36:54,176 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:36:54,176 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:36:54,177 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:36:54,181 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/default/TestLogRolling-testLogRollOnPipelineRestart/f1e14f21609a5d55957d236c91429d5d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-16T16:36:54,182 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-16T16:36:54,182 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:54,182 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f1e14f21609a5d55957d236c91429d5d:
Waiting for close lock at 1731775014176Running coprocessor pre-close hooks at 1731775014176Disabling compacts and flushes for region at 1731775014176Disabling writes for close at 1731775014176Writing region close event to WAL at 1731775014176Running coprocessor post-close hooks at 1731775014182 (+6 ms)Closed at 1731775014182
2024-11-16T16:36:54,182 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731774989072.f1e14f21609a5d55957d236c91429d5d.
2024-11-16T16:36:54,182 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:36:54,183 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:36:54,183 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775014176Running coprocessor pre-close hooks at 1731775014176Disabling compacts and flushes for region at 1731775014176Disabling writes for close at 1731775014177 (+1 ms)Writing region close event to WAL at 1731775014178 (+1 ms)Running coprocessor post-close hooks at 1731775014182 (+4 ms)Closed at 1731775014182
2024-11-16T16:36:54,183 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:36:54,321 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:54,348 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T16:36:54,348 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T16:36:54,376 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,46713,1731774987814; all regions closed.
2024-11-16T16:36:54,377 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,377 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,377 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,377 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,378 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:54,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741840_1023 (size=825)
2024-11-16T16:36:54,380 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741840_1023 (size=825)
2024-11-16T16:36:54,464 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:54,655 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:55,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:55,656 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:56,465 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:56,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:56,800 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-16T16:36:57,466 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:57,605 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:36:57,657 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:57,977 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta after 4001ms
2024-11-16T16:36:57,978 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/WALs/303fe01965e8,46713,1731774987814/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta to hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs/303fe01965e8%2C46713%2C1731774987814.meta.1731774988864.meta
2024-11-16T16:36:57,982 DEBUG [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs
2024-11-16T16:36:57,982 INFO  [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C46713%2C1731774987814.meta:.meta(num 1731775013967)
2024-11-16T16:36:57,983 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:57,983 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:57,983 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:57,983 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:57,983 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:57,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741844_1028 (size=1162)
2024-11-16T16:36:57,986 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741844_1028 (size=1162)
2024-11-16T16:36:57,992 DEBUG [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs
2024-11-16T16:36:57,992 INFO  [RS:0;303fe01965e8:46713 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C46713%2C1731774987814:(num 1731775014089)
2024-11-16T16:36:57,992 DEBUG [RS:0;303fe01965e8:46713 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:36:57,992 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:36:57,992 INFO  [RS:0;303fe01965e8:46713 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:36:57,993 INFO  [RS:0;303fe01965e8:46713 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T16:36:57,993 INFO  [RS:0;303fe01965e8:46713 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:36:57,993 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:36:57,993 INFO  [RS:0;303fe01965e8:46713 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46713
2024-11-16T16:36:58,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:36:58,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,46713,1731774987814
2024-11-16T16:36:58,021 INFO  [RS:0;303fe01965e8:46713 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:36:58,031 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,46713,1731774987814]
2024-11-16T16:36:58,042 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,46713,1731774987814 already deleted, retry=false
2024-11-16T16:36:58,042 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,46713,1731774987814 expired; onlineServers=0
2024-11-16T16:36:58,042 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,45601,1731774987626' *****
2024-11-16T16:36:58,042 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:36:58,042 INFO  [M:0;303fe01965e8:45601 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:36:58,042 INFO  [M:0;303fe01965e8:45601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:36:58,042 DEBUG [M:0;303fe01965e8:45601 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:36:58,042 DEBUG [M:0;303fe01965e8:45601 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:36:58,042 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:36:58,042 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774988185 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731774988185,5,FailOnTimeoutGroup]
2024-11-16T16:36:58,042 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774988185 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731774988185,5,FailOnTimeoutGroup]
2024-11-16T16:36:58,042 INFO  [M:0;303fe01965e8:45601 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:36:58,043 INFO  [M:0;303fe01965e8:45601 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:36:58,043 DEBUG [M:0;303fe01965e8:45601 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:36:58,043 INFO  [M:0;303fe01965e8:45601 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:36:58,043 INFO  [M:0;303fe01965e8:45601 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:36:58,043 INFO  [M:0;303fe01965e8:45601 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:36:58,043 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:36:58,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:36:58,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:36:58,052 DEBUG [M:0;303fe01965e8:45601 {}] zookeeper.ZKUtil(347): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:36:58,052 WARN  [M:0;303fe01965e8:45601 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:36:58,053 INFO  [M:0;303fe01965e8:45601 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/.lastflushedseqids
2024-11-16T16:36:58,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741846_1030 (size=111)
2024-11-16T16:36:58,059 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741846_1030 (size=111)
2024-11-16T16:36:58,059 INFO  [M:0;303fe01965e8:45601 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:36:58,059 INFO  [M:0;303fe01965e8:45601 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:36:58,059 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:36:58,059 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:58,059 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:58,059 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:36:58,059 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:58,060 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB
2024-11-16T16:36:58,060 ERROR [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData-prefix:303fe01965e8,45601,1731774987626 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:58,060 WARN  [FSHLog-0-hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData-prefix:303fe01965e8,45601,1731774987626 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:58,060 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog 303fe01965e8%2C45601%2C1731774987626:(num 1731774987966) roll requested
2024-11-16T16:36:58,060 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C45601%2C1731774987626.1731775018060
2024-11-16T16:36:58,065 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,065 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,066 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,066 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,066 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,066 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731775018060
2024-11-16T16:36:58,066 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:58,066 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33515,DS-df86c6c4-1300-4aed-afd1-b0012be56f63,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-16T16:36:58,066 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966
2024-11-16T16:36:58,067 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39815:39815),(127.0.0.1/127.0.0.1:34811:34811)]
2024-11-16T16:36:58,067 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 is not closed yet, will try archiving it next time
2024-11-16T16:36:58,067 WARN  [IPC Server handler 0 on default port 44217 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-16T16:36:58,067 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 after 1ms
2024-11-16T16:36:58,084 DEBUG [M:0;303fe01965e8:45601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f1a5cd23800f4f16bc5856a304f32ec9 is 82, key is hbase:meta,,1/info:regioninfo/1731774988900/Put/seqid=0
2024-11-16T16:36:58,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741848_1033 (size=5672)
2024-11-16T16:36:58,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741848_1033 (size=5672)
2024-11-16T16:36:58,090 INFO  [M:0;303fe01965e8:45601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f1a5cd23800f4f16bc5856a304f32ec9
2024-11-16T16:36:58,112 DEBUG [M:0;303fe01965e8:45601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/661d4180cc2c48959c72a75c6e307cde is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731774989453/Put/seqid=0
2024-11-16T16:36:58,117 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741849_1034 (size=6118)
2024-11-16T16:36:58,118 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741849_1034 (size=6118)
2024-11-16T16:36:58,118 INFO  [M:0;303fe01965e8:45601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/661d4180cc2c48959c72a75c6e307cde
2024-11-16T16:36:58,131 INFO  [RS:0;303fe01965e8:46713 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:36:58,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:58,132 INFO  [RS:0;303fe01965e8:46713 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,46713,1731774987814; zookeeper connection closed.
2024-11-16T16:36:58,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46713-0x101448139040001, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:36:58,132 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1129ff57 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1129ff57
2024-11-16T16:36:58,132 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:36:58,140 DEBUG [M:0;303fe01965e8:45601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1dfb350e9d644375b9303fbb815d7754 is 69, key is 303fe01965e8,46713,1731774987814/rs:state/1731774988289/Put/seqid=0
2024-11-16T16:36:58,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741850_1035 (size=5156)
2024-11-16T16:36:58,145 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741850_1035 (size=5156)
2024-11-16T16:36:58,146 INFO  [M:0;303fe01965e8:45601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1dfb350e9d644375b9303fbb815d7754
2024-11-16T16:36:58,170 DEBUG [M:0;303fe01965e8:45601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/33d575e1b15e4668acf0fa3be2b475bd is 52, key is load_balancer_on/state:d/1731774989066/Put/seqid=0
2024-11-16T16:36:58,175 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741851_1036 (size=5056)
2024-11-16T16:36:58,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741851_1036 (size=5056)
2024-11-16T16:36:58,176 INFO  [M:0;303fe01965e8:45601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/33d575e1b15e4668acf0fa3be2b475bd
2024-11-16T16:36:58,182 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f1a5cd23800f4f16bc5856a304f32ec9 as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f1a5cd23800f4f16bc5856a304f32ec9
2024-11-16T16:36:58,187 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f1a5cd23800f4f16bc5856a304f32ec9, entries=8, sequenceid=56, filesize=5.5 K
2024-11-16T16:36:58,189 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/661d4180cc2c48959c72a75c6e307cde as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/661d4180cc2c48959c72a75c6e307cde
2024-11-16T16:36:58,194 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/661d4180cc2c48959c72a75c6e307cde, entries=6, sequenceid=56, filesize=6.0 K
2024-11-16T16:36:58,195 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1dfb350e9d644375b9303fbb815d7754 as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1dfb350e9d644375b9303fbb815d7754
2024-11-16T16:36:58,201 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1dfb350e9d644375b9303fbb815d7754, entries=1, sequenceid=56, filesize=5.0 K
2024-11-16T16:36:58,202 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/33d575e1b15e4668acf0fa3be2b475bd as hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/33d575e1b15e4668acf0fa3be2b475bd
2024-11-16T16:36:58,208 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/33d575e1b15e4668acf0fa3be2b475bd, entries=1, sequenceid=56, filesize=4.9 K
2024-11-16T16:36:58,209 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 150ms, sequenceid=56, compaction requested=false
2024-11-16T16:36:58,210 INFO  [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:36:58,210 DEBUG [M:0;303fe01965e8:45601 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775018059Disabling compacts and flushes for region at 1731775018059Disabling writes for close at 1731775018059Obtaining lock to block concurrent updates at 1731775018060 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731775018060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731775018060Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731775018067 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731775018067Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731775018083 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731775018083Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731775018096 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731775018111 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731775018111Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731775018123 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731775018140 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731775018140Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731775018151 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731775018169 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731775018169Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@225e90ba: reopening flushed file at 1731775018181 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53507fe: reopening flushed file at 1731775018188 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d5b92a2: reopening flushed file at 1731775018194 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58786fc6: reopening flushed file at 1731775018201 (+7 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 150ms, sequenceid=56, compaction requested=false at 1731775018209 (+8 ms)Writing region close event to WAL at 1731775018210 (+1 ms)Closed at 1731775018210
2024-11-16T16:36:58,211 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,211 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,211 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,211 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,211 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:36:58,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44855 is added to blk_1073741847_1031 (size=757)
2024-11-16T16:36:58,213 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39091 is added to blk_1073741847_1031 (size=757)
2024-11-16T16:36:58,467 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:58,658 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:59,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,183 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,207 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,208 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,212 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,213 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,215 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,219 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,219 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:59,659 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:36:59,722 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:36:59,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,723 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,743 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,744 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,745 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,749 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,752 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:36:59,800 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-16T16:37:00,468 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:00,660 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:37:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:37:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-16T16:37:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-16T16:37:01,469 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:01,661 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:02,068 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 after 4002ms
2024-11-16T16:37:02,068 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/WALs/303fe01965e8,45601,1731774987626/303fe01965e8%2C45601%2C1731774987626.1731774987966 to hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/oldWALs/303fe01965e8%2C45601%2C1731774987626.1731774987966
2024-11-16T16:37:02,072 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/MasterData/oldWALs/303fe01965e8%2C45601%2C1731774987626.1731774987966 to hdfs://localhost:44217/user/jenkins/test-data/e03f2111-bfc8-f921-3719-e0c373a2574e/oldWALs/303fe01965e8%2C45601%2C1731774987626.1731774987966$masterlocalwal$
2024-11-16T16:37:02,072 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:37:02,072 INFO  [M:0;303fe01965e8:45601 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:37:02,072 INFO  [M:0;303fe01965e8:45601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45601
2024-11-16T16:37:02,073 INFO  [M:0;303fe01965e8:45601 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:37:02,231 INFO  [M:0;303fe01965e8:45601 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:37:02,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:02,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45601-0x101448139040000, quorum=127.0.0.1:62941, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:02,234 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@478c28d0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:02,234 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8cbc01a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:02,235 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:02,235 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f2d69a0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:02,235 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a0c1f66{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:02,237 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:37:02,237 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:37:02,237 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 1f28b283-37b1-4694-903c-2a183702a7cd) service to localhost/127.0.0.1:44217
2024-11-16T16:37:02,237 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:37:02,238 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data3/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:02,238 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data4/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:02,238 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:37:02,252 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10e0b03{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:02,252 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3c894b0d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:02,252 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:02,253 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@686dd8bf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:02,253 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48f4fa87{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:02,254 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:37:02,254 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:37:02,254 WARN  [BP-2119659671-172.17.0.2-1731774985124 heartbeating to localhost/127.0.0.1:44217 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2119659671-172.17.0.2-1731774985124 (Datanode Uuid 6f116c78-038d-46f0-946d-fce3c8fe505a) service to localhost/127.0.0.1:44217
2024-11-16T16:37:02,254 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:37:02,255 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data1/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:02,255 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/cluster_940619cb-bf12-cbe4-8d33-9381e22f9ec4/data/data2/current/BP-2119659671-172.17.0.2-1731774985124 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:02,255 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:37:02,260 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@585ec7cf{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:37:02,261 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@27bc6094{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:02,261 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:02,262 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@209ebed6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:02,262 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@115eaf6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:02,268 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:37:02,286 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:37:02,296 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=183 (was 157)
Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44217 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44217
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44217 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44217 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:44217
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44217
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 431) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 174) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3553 (was 3796)
2024-11-16T16:37:02,303 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=183, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=3554
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.log.dir so I do NOT create it in target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c2941a4-d976-7235-c028-d9707460360b/hadoop.tmp.dir so I do NOT create it in target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76, deleteOnExit=true
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:37:02,304 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/test.cache.data in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:37:02,305 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:37:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:37:02,306 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:37:02,307 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:37:02,325 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:37:02,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:02,661 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:02,969 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:02,974 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:02,985 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:02,985 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:02,985 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:37:02,987 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:02,987 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a3a01a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:02,988 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75c50826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:03,094 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6694026d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/java.io.tmpdir/jetty-localhost-35469-hadoop-hdfs-3_4_1-tests_jar-_-any-59452043895468305/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:37:03,094 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2bd70390{HTTP/1.1, (http/1.1)}{localhost:35469}
2024-11-16T16:37:03,094 INFO  [Time-limited test {}] server.Server(415): Started @198870ms
2024-11-16T16:37:03,111 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:37:03,377 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:03,381 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:03,382 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:03,382 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:03,382 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:37:03,382 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46bd2b1a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:03,382 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40a21084{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:03,470 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:03,486 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3272a875{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/java.io.tmpdir/jetty-localhost-34277-hadoop-hdfs-3_4_1-tests_jar-_-any-17755159730435294078/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:03,487 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@61918bd4{HTTP/1.1, (http/1.1)}{localhost:34277}
2024-11-16T16:37:03,487 INFO  [Time-limited test {}] server.Server(415): Started @199263ms
2024-11-16T16:37:03,489 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:37:03,543 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:03,548 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:03,558 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:03,558 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:03,558 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:37:03,559 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33307a55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:03,559 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acc16de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:03,662 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:03,668 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3d01a0e4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/java.io.tmpdir/jetty-localhost-36671-hadoop-hdfs-3_4_1-tests_jar-_-any-1340867473777486799/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:03,668 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166de790{HTTP/1.1, (http/1.1)}{localhost:36671}
2024-11-16T16:37:03,668 INFO  [Time-limited test {}] server.Server(415): Started @199444ms
2024-11-16T16:37:03,669 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:37:04,471 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:04,614 WARN  [Thread-1667 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data1/current/BP-1894627294-172.17.0.2-1731775022339/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:04,614 WARN  [Thread-1668 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data2/current/BP-1894627294-172.17.0.2-1731775022339/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:04,634 WARN  [Thread-1631 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:37:04,636 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd662f06c6f211522 with lease ID 0x5736dc03da1b16cd: Processing first storage report for DS-4d4a1908-ff09-4018-8fbb-806e01f001c7 from datanode DatanodeRegistration(127.0.0.1:35931, datanodeUuid=99dcb861-7903-49d5-8b0b-3c91babd2180, infoPort=39987, infoSecurePort=0, ipcPort=36009, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339)
2024-11-16T16:37:04,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd662f06c6f211522 with lease ID 0x5736dc03da1b16cd: from storage DS-4d4a1908-ff09-4018-8fbb-806e01f001c7 node DatanodeRegistration(127.0.0.1:35931, datanodeUuid=99dcb861-7903-49d5-8b0b-3c91babd2180, infoPort=39987, infoSecurePort=0, ipcPort=36009, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:04,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd662f06c6f211522 with lease ID 0x5736dc03da1b16cd: Processing first storage report for DS-e6f5366b-b18a-47d9-a72e-2091d5c8e2ba from datanode DatanodeRegistration(127.0.0.1:35931, datanodeUuid=99dcb861-7903-49d5-8b0b-3c91babd2180, infoPort=39987, infoSecurePort=0, ipcPort=36009, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339)
2024-11-16T16:37:04,637 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd662f06c6f211522 with lease ID 0x5736dc03da1b16cd: from storage DS-e6f5366b-b18a-47d9-a72e-2091d5c8e2ba node DatanodeRegistration(127.0.0.1:35931, datanodeUuid=99dcb861-7903-49d5-8b0b-3c91babd2180, infoPort=39987, infoSecurePort=0, ipcPort=36009, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:04,663 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:04,835 WARN  [Thread-1678 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data3/current/BP-1894627294-172.17.0.2-1731775022339/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:04,835 WARN  [Thread-1679 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data4/current/BP-1894627294-172.17.0.2-1731775022339/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:04,855 WARN  [Thread-1654 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:37:04,857 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x81fd940554552c15 with lease ID 0x5736dc03da1b16ce: Processing first storage report for DS-ea3d92aa-d9e1-4bc5-bebd-66ed66d755d7 from datanode DatanodeRegistration(127.0.0.1:45701, datanodeUuid=c799d081-c560-4e53-b335-73d91d53dd56, infoPort=36347, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339)
2024-11-16T16:37:04,857 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x81fd940554552c15 with lease ID 0x5736dc03da1b16ce: from storage DS-ea3d92aa-d9e1-4bc5-bebd-66ed66d755d7 node DatanodeRegistration(127.0.0.1:45701, datanodeUuid=c799d081-c560-4e53-b335-73d91d53dd56, infoPort=36347, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:04,857 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x81fd940554552c15 with lease ID 0x5736dc03da1b16ce: Processing first storage report for DS-e5e2b2d4-4b2e-4097-afa9-de1d5b26a4a3 from datanode DatanodeRegistration(127.0.0.1:45701, datanodeUuid=c799d081-c560-4e53-b335-73d91d53dd56, infoPort=36347, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339)
2024-11-16T16:37:04,857 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x81fd940554552c15 with lease ID 0x5736dc03da1b16ce: from storage DS-e5e2b2d4-4b2e-4097-afa9-de1d5b26a4a3 node DatanodeRegistration(127.0.0.1:45701, datanodeUuid=c799d081-c560-4e53-b335-73d91d53dd56, infoPort=36347, infoSecurePort=0, ipcPort=40937, storageInfo=lv=-57;cid=testClusterID;nsid=227490427;c=1731775022339), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:04,902 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d
2024-11-16T16:37:04,905 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/zookeeper_0, clientPort=64939, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:37:04,906 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64939
2024-11-16T16:37:04,906 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:04,908 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:04,917 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:37:04,918 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:37:04,919 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794 with version=8
2024-11-16T16:37:04,919 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:37:04,921 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:37:04,922 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37995
2024-11-16T16:37:04,923 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37995 connecting to ZooKeeper ensemble=127.0.0.1:64939
2024-11-16T16:37:04,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379950x0, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:37:04,981 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37995-0x1014481cab30000 connected
2024-11-16T16:37:05,066 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:05,067 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:05,070 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:37:05,070 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794, hbase.cluster.distributed=false
2024-11-16T16:37:05,072 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:37:05,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37995
2024-11-16T16:37:05,072 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37995
2024-11-16T16:37:05,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37995
2024-11-16T16:37:05,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37995
2024-11-16T16:37:05,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37995
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:37:05,093 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:37:05,094 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42251
2024-11-16T16:37:05,096 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42251 connecting to ZooKeeper ensemble=127.0.0.1:64939
2024-11-16T16:37:05,096 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:05,098 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:05,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:422510x0, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:37:05,108 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:422510x0, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:37:05,108 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42251-0x1014481cab30001 connected
2024-11-16T16:37:05,108 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:37:05,109 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:37:05,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:37:05,110 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:37:05,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-16T16:37:05,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42251
2024-11-16T16:37:05,111 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42251
2024-11-16T16:37:05,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-16T16:37:05,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42251
2024-11-16T16:37:05,126 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:37995
2024-11-16T16:37:05,126 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:37:05,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:37:05,137 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:37:05,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,150 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:37:05,151 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,37995,1731775024920 from backup master directory
2024-11-16T16:37:05,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:37:05,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:37:05,160 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:37:05,160 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,165 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/hbase.id] with ID: f1698b10-79b0-48dd-8fe2-3d082851bef6
2024-11-16T16:37:05,165 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/.tmp/hbase.id
2024-11-16T16:37:05,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:37:05,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:37:05,172 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/.tmp/hbase.id]:[hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/hbase.id]
2024-11-16T16:37:05,183 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:05,183 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:37:05,185 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-16T16:37:05,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,198 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:37:05,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:37:05,199 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:37:05,200 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:37:05,200 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:37:05,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:37:05,208 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:37:05,209 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store
2024-11-16T16:37:05,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:37:05,217 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:37:05,218 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:05,218 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:05,218 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775025218Disabling compacts and flushes for region at 1731775025218Disabling writes for close at 1731775025218Writing region close event to WAL at 1731775025218Closed at 1731775025218
2024-11-16T16:37:05,219 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/.initializing
2024-11-16T16:37:05,220 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/WALs/303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,223 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C37995%2C1731775024920, suffix=, logDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/WALs/303fe01965e8,37995,1731775024920, archiveDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/oldWALs, maxLogs=10
2024-11-16T16:37:05,223 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C37995%2C1731775024920.1731775025223
2024-11-16T16:37:05,229 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/WALs/303fe01965e8,37995,1731775024920/303fe01965e8%2C37995%2C1731775024920.1731775025223
2024-11-16T16:37:05,231 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36347:36347),(127.0.0.1/127.0.0.1:39987:39987)]
2024-11-16T16:37:05,235 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:37:05,235 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:05,235 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,235 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,237 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:37:05,238 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:37:05,238 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,239 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:05,239 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:37:05,240 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:37:05,240 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,240 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:37:05,240 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:37:05,241 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:37:05,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,242 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:37:05,242 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:37:05,243 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:37:05,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,243 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:37:05,243 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,244 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,244 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,246 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,246 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,247 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:37:05,248 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:37:05,249 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:37:05,250 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=866387, jitterRate=0.10166843235492706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:37:05,250 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731775025235Initializing all the Stores at 1731775025236 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775025236Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775025237 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775025237Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775025237Cleaning up temporary data from old regions at 1731775025246 (+9 ms)Region opened successfully at 1731775025250 (+4 ms)
2024-11-16T16:37:05,251 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:37:05,253 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f56efb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:37:05,254 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:37:05,255 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:37:05,255 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:37:05,255 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:37:05,256 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:37:05,256 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:37:05,256 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:37:05,259 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:37:05,260 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:37:05,265 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:37:05,266 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:37:05,267 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:37:05,276 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:37:05,276 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:37:05,277 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:37:05,286 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:37:05,287 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:37:05,297 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:37:05,300 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:37:05,308 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:37:05,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:37:05,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:37:05,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,319 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,37995,1731775024920, sessionid=0x1014481cab30000, setting cluster-up flag (Was=false)
2024-11-16T16:37:05,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,371 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:37:05,372 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,423 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:37:05,425 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,37995,1731775024920
2024-11-16T16:37:05,426 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:37:05,428 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:37:05,428 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:37:05,429 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:37:05,429 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,37995,1731775024920
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:37:05,430 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:37:05,431 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731775055432
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:37:05,432 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:37:05,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:37:05,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:37:05,433 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:37:05,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:37:05,433 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:37:05,433 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:37:05,434 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:37:05,434 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775025434,5,FailOnTimeoutGroup]
2024-11-16T16:37:05,434 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775025434,5,FailOnTimeoutGroup]
2024-11-16T16:37:05,434 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,434 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:37:05,434 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,434 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,434 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,434 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:37:05,441 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:37:05,442 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:37:05,443 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:37:05,443 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794
2024-11-16T16:37:05,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:37:05,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:37:05,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:05,452 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:37:05,453 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:37:05,453 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,454 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:05,454 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:37:05,455 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:37:05,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,456 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:05,456 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:37:05,457 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:37:05,457 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,458 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:05,458 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:37:05,459 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:37:05,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:05,460 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:05,460 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:37:05,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740
2024-11-16T16:37:05,461 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740
2024-11-16T16:37:05,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:37:05,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:37:05,463 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:37:05,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:37:05,466 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:37:05,467 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=818129, jitterRate=0.040305137634277344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:37:05,467 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731775025451Initializing all the Stores at 1731775025451Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775025451Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775025452 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775025452Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775025452Cleaning up temporary data from old regions at 1731775025463 (+11 ms)Region opened successfully at 1731775025467 (+4 ms)
2024-11-16T16:37:05,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:37:05,468 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:37:05,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:37:05,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:37:05,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:37:05,468 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:37:05,468 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775025467Disabling compacts and flushes for region at 1731775025467Disabling writes for close at 1731775025468 (+1 ms)Writing region close event to WAL at 1731775025468Closed at 1731775025468
2024-11-16T16:37:05,469 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:37:05,469 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:37:05,470 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:37:05,471 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:37:05,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:05,472 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:37:05,514 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(746): ClusterId : f1698b10-79b0-48dd-8fe2-3d082851bef6
2024-11-16T16:37:05,514 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:37:05,527 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:37:05,527 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:37:05,540 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:37:05,540 DEBUG [RS:0;303fe01965e8:42251 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@713726c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:37:05,552 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:42251
2024-11-16T16:37:05,553 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:37:05,553 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:37:05,553 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:37:05,553 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,37995,1731775024920 with port=42251, startcode=1731775025092
2024-11-16T16:37:05,554 DEBUG [RS:0;303fe01965e8:42251 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:37:05,555 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:37:05,556 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,556 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37995 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,557 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794
2024-11-16T16:37:05,557 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37859
2024-11-16T16:37:05,557 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:37:05,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:37:05,569 DEBUG [RS:0;303fe01965e8:42251 {}] zookeeper.ZKUtil(111): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,569 WARN  [RS:0;303fe01965e8:42251 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:37:05,569 INFO  [RS:0;303fe01965e8:42251 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:37:05,569 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,569 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,42251,1731775025092]
2024-11-16T16:37:05,573 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:37:05,574 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:37:05,574 INFO  [RS:0;303fe01965e8:42251 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:37:05,574 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,575 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:37:05,575 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:37:05,575 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:37:05,576 DEBUG [RS:0;303fe01965e8:42251 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,577 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,42251,1731775025092-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:37:05,592 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:37:05,592 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,42251,1731775025092-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,592 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,592 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.Replication(171): 303fe01965e8,42251,1731775025092 started
2024-11-16T16:37:05,609 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:05,609 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,42251,1731775025092, RpcServer on 303fe01965e8/172.17.0.2:42251, sessionid=0x1014481cab30001
2024-11-16T16:37:05,609 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:37:05,609 DEBUG [RS:0;303fe01965e8:42251 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,609 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,42251,1731775025092'
2024-11-16T16:37:05,609 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,42251,1731775025092'
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:37:05,610 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:37:05,611 DEBUG [RS:0;303fe01965e8:42251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:37:05,611 INFO  [RS:0;303fe01965e8:42251 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:37:05,611 INFO  [RS:0;303fe01965e8:42251 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:37:05,622 WARN  [303fe01965e8:37995 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:37:05,663 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:05,713 INFO  [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C42251%2C1731775025092, suffix=, logDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092, archiveDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs, maxLogs=32
2024-11-16T16:37:05,713 INFO  [RS:0;303fe01965e8:42251 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C42251%2C1731775025092.1731775025713
2024-11-16T16:37:05,719 INFO  [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775025713
2024-11-16T16:37:05,720 DEBUG [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36347:36347),(127.0.0.1/127.0.0.1:39987:39987)]
2024-11-16T16:37:05,873 DEBUG [303fe01965e8:37995 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:37:05,873 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,42251,1731775025092
2024-11-16T16:37:05,875 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,42251,1731775025092, state=OPENING
2024-11-16T16:37:05,887 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:37:05,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:05,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:37:05,972 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:37:05,972 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:37:05,972 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,42251,1731775025092}]
2024-11-16T16:37:06,125 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:37:06,128 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35441, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:37:06,134 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:37:06,134 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:37:06,136 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C42251%2C1731775025092.meta, suffix=.meta, logDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092, archiveDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs, maxLogs=32
2024-11-16T16:37:06,137 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C42251%2C1731775025092.meta.1731775026137.meta
2024-11-16T16:37:06,142 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.meta.1731775026137.meta
2024-11-16T16:37:06,143 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39987:39987),(127.0.0.1/127.0.0.1:36347:36347)]
2024-11-16T16:37:06,144 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:37:06,145 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:37:06,145 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:37:06,147 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:37:06,148 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:37:06,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:06,148 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:37:06,149 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:37:06,149 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:06,150 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:37:06,151 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:37:06,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:06,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:37:06,153 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:37:06,153 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,153 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:37:06,153 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:37:06,154 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740
2024-11-16T16:37:06,155 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740
2024-11-16T16:37:06,157 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:37:06,157 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:37:06,158 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:37:06,159 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:37:06,160 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841162, jitterRate=0.0695936381816864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:37:06,160 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:37:06,161 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731775026146Writing region info on filesystem at 1731775026146Initializing all the Stores at 1731775026146Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775026146Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775026147 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775026147Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775026147Cleaning up temporary data from old regions at 1731775026157 (+10 ms)Running coprocessor post-open hooks at 1731775026160 (+3 ms)Region opened successfully at 1731775026161 (+1 ms)
2024-11-16T16:37:06,163 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731775026124
2024-11-16T16:37:06,165 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:37:06,165 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:37:06,166 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,42251,1731775025092
2024-11-16T16:37:06,167 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,42251,1731775025092, state=OPEN
2024-11-16T16:37:06,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:37:06,209 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,42251,1731775025092
2024-11-16T16:37:06,209 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:37:06,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:37:06,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:37:06,213 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:37:06,213 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,42251,1731775025092 in 237 msec
2024-11-16T16:37:06,217 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:37:06,217 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 743 msec
2024-11-16T16:37:06,218 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:37:06,218 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:37:06,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:37:06,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,42251,1731775025092, seqNum=-1]
2024-11-16T16:37:06,220 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:37:06,222 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:37:06,228 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 798 msec
2024-11-16T16:37:06,228 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731775026228, completionTime=-1
2024-11-16T16:37:06,228 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:37:06,228 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731775086230
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775146230
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,230 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,231 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:37995, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,231 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,231 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,233 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.073sec
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:37:06,235 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:37:06,237 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:37:06,237 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:37:06,237 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37995,1731775024920-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:06,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781dc17d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:37:06,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,37995,-1 for getting cluster id
2024-11-16T16:37:06,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:37:06,316 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f1698b10-79b0-48dd-8fe2-3d082851bef6'
2024-11-16T16:37:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:37:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f1698b10-79b0-48dd-8fe2-3d082851bef6"

2024-11-16T16:37:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d4168a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:37:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,37995,-1]
2024-11-16T16:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:37:06,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:06,319 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:37:06,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46380c5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:37:06,321 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:37:06,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,42251,1731775025092, seqNum=-1]
2024-11-16T16:37:06,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:37:06,324 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:37:06,326 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,37995,1731775024920
2024-11-16T16:37:06,326 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:37:06,329 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:37:06,329 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T16:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,37995,1731775024920
2024-11-16T16:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7df76e7c
2024-11-16T16:37:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T16:37:06,331 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T16:37:06,332 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-16T16:37:06,332 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-16T16:37:06,332 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:37:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:06,335 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T16:37:06,335 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,335 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-16T16:37:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:37:06,336 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T16:37:06,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741835_1011 (size=405)
2024-11-16T16:37:06,342 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741835_1011 (size=405)
2024-11-16T16:37:06,345 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 64c8f3425b0e0c591a6235842242a340, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794
2024-11-16T16:37:06,351 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741836_1012 (size=88)
2024-11-16T16:37:06,352 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741836_1012 (size=88)
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 64c8f3425b0e0c591a6235842242a340, disabling compactions & flushes
2024-11-16T16:37:06,357 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340. after waiting 0 ms
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,357 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,357 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 64c8f3425b0e0c591a6235842242a340:
Waiting for close lock at 1731775026357Disabling compacts and flushes for region at 1731775026357Disabling writes for close at 1731775026357Writing region close event to WAL at 1731775026357Closed at 1731775026357
2024-11-16T16:37:06,359 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T16:37:06,359 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731775026359"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731775026359"}]},"ts":"1731775026359"}
2024-11-16T16:37:06,362 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T16:37:06,363 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T16:37:06,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731775026363"}]},"ts":"1731775026363"}
2024-11-16T16:37:06,365 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-16T16:37:06,366 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c8f3425b0e0c591a6235842242a340, ASSIGN}]
2024-11-16T16:37:06,367 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c8f3425b0e0c591a6235842242a340, ASSIGN
2024-11-16T16:37:06,368 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c8f3425b0e0c591a6235842242a340, ASSIGN; state=OFFLINE, location=303fe01965e8,42251,1731775025092; forceNewPlan=false, retain=false
2024-11-16T16:37:06,472 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:06,519 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64c8f3425b0e0c591a6235842242a340, regionState=OPENING, regionLocation=303fe01965e8,42251,1731775025092
2024-11-16T16:37:06,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c8f3425b0e0c591a6235842242a340, ASSIGN because future has completed
2024-11-16T16:37:06,523 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c8f3425b0e0c591a6235842242a340, server=303fe01965e8,42251,1731775025092}]
2024-11-16T16:37:06,664 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:06,681 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,681 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 64c8f3425b0e0c591a6235842242a340, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:37:06,681 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,681 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:37:06,682 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,682 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,683 INFO  [StoreOpener-64c8f3425b0e0c591a6235842242a340-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 64c8f3425b0e0c591a6235842242a340 
2024-11-16T16:37:06,684 INFO  [StoreOpener-64c8f3425b0e0c591a6235842242a340-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64c8f3425b0e0c591a6235842242a340 columnFamilyName info
2024-11-16T16:37:06,685 DEBUG [StoreOpener-64c8f3425b0e0c591a6235842242a340-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:37:06,685 INFO  [StoreOpener-64c8f3425b0e0c591a6235842242a340-1 {}] regionserver.HStore(327): Store=64c8f3425b0e0c591a6235842242a340/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:37:06,685 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,686 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,687 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,687 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,687 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,689 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,692 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:37:06,692 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:37:06,692 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 64c8f3425b0e0c591a6235842242a340; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=768403, jitterRate=-0.0229262113571167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:37:06,693 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:06,693 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,693 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 64c8f3425b0e0c591a6235842242a340:
Running coprocessor pre-open hook at 1731775026682Writing region info on filesystem at 1731775026682Initializing all the Stores at 1731775026683 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775026683Cleaning up temporary data from old regions at 1731775026687 (+4 ms)Running coprocessor post-open hooks at 1731775026693 (+6 ms)Region opened successfully at 1731775026693
2024-11-16T16:37:06,694 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,694 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,694 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,694 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340., pid=6, masterSystemTime=1731775026677
2024-11-16T16:37:06,695 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,695 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,697 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,697 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:06,699 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=64c8f3425b0e0c591a6235842242a340, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,42251,1731775025092
2024-11-16T16:37:06,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64c8f3425b0e0c591a6235842242a340, server=303fe01965e8,42251,1731775025092 because future has completed
2024-11-16T16:37:06,705 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T16:37:06,706 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 64c8f3425b0e0c591a6235842242a340, server=303fe01965e8,42251,1731775025092 in 179 msec
2024-11-16T16:37:06,708 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T16:37:06,708 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=64c8f3425b0e0c591a6235842242a340, ASSIGN in 340 msec
2024-11-16T16:37:06,709 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T16:37:06,710 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731775026709"}]},"ts":"1731775026709"}
2024-11-16T16:37:06,712 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-16T16:37:06,713 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T16:37:06,715 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 381 msec
2024-11-16T16:37:06,727 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,728 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,729 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,729 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,735 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,736 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:06,740 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:07,473 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:07,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:08,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:08,665 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:09,474 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:09,666 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:10,475 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:10,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:11,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:11,649 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:37:11,650 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,650 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,651 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,667 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:11,675 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,675 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,676 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,676 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,676 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,676 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,680 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,683 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:37:11,690 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:37:11,690 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T16:37:11,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:37:11,691 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T16:37:11,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-16T16:37:11,691 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-16T16:37:11,691 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:11,691 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-16T16:37:11,692 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:37:11,693 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-16T16:37:12,476 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:12,668 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:13,477 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:13,669 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:14,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:14,670 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:15,478 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:15,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:37:16,357 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-16T16:37:16,357 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-16T16:37:16,361 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:16,361 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:16,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340., hostname=303fe01965e8,42251,1731775025092, seqNum=2]
2024-11-16T16:37:16,374 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:16,381 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-16T16:37:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-16T16:37:16,383 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-16T16:37:16,384 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-16T16:37:16,479 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:16,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-16T16:37:16,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:16,549 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 64c8f3425b0e0c591a6235842242a340 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:37:16,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/53479e6e3015444c9b06892f4924d567 is 1080, key is row0001/info:/1731775036366/Put/seqid=0
2024-11-16T16:37:16,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741837_1013 (size=6033)
2024-11-16T16:37:16,573 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741837_1013 (size=6033)
2024-11-16T16:37:16,574 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/53479e6e3015444c9b06892f4924d567
2024-11-16T16:37:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/53479e6e3015444c9b06892f4924d567 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567
2024-11-16T16:37:16,587 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567, entries=1, sequenceid=5, filesize=5.9 K
2024-11-16T16:37:16,588 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 39ms, sequenceid=5, compaction requested=false
2024-11-16T16:37:16,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 64c8f3425b0e0c591a6235842242a340:

2024-11-16T16:37:16,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:16,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-16T16:37:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-16T16:37:16,597 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-16T16:37:16,597 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec
2024-11-16T16:37:16,600 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 222 msec
2024-11-16T16:37:16,671 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:17,480 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:17,672 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:18,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:18,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:19,481 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:19,673 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:20,482 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:20,674 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:21,483 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:21,675 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:22,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:22,484 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta after 68059ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:37:22,676 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:23,485 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:23,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:24,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:24,677 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:25,486 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:25,678 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-16T16:37:26,457 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-16T16:37:26,460 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-16T16:37:26,463 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-16T16:37:26,464 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-16T16:37:26,465 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-16T16:37:26,487 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:26,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-16T16:37:26,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:26,620 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 64c8f3425b0e0c591a6235842242a340 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:37:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/f38bcc43375f4831a3c17521deffee01 is 1080, key is row0002/info:/1731775046458/Put/seqid=0
2024-11-16T16:37:26,637 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741838_1014 (size=6033)
2024-11-16T16:37:26,638 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741838_1014 (size=6033)
2024-11-16T16:37:26,638 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/f38bcc43375f4831a3c17521deffee01
2024-11-16T16:37:26,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/f38bcc43375f4831a3c17521deffee01 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01
2024-11-16T16:37:26,653 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01, entries=1, sequenceid=9, filesize=5.9 K
2024-11-16T16:37:26,654 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 34ms, sequenceid=9, compaction requested=false
2024-11-16T16:37:26,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 64c8f3425b0e0c591a6235842242a340:

2024-11-16T16:37:26,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:26,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-16T16:37:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-16T16:37:26,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-16T16:37:26,658 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 191 msec
2024-11-16T16:37:26,660 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 199 msec
2024-11-16T16:37:26,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:27,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:27,679 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:27,680 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277 after 68063ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-16T16:37:28,488 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:28,681 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:29,489 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:29,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:30,490 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:30,682 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:31,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:31,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:32,491 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:32,683 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:33,492 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:33,684 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:34,493 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:34,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:34,902 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:37:35,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:35,685 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:36,494 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-16T16:37:36,528 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-16T16:37:36,532 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C42251%2C1731775025092.1731775056532
2024-11-16T16:37:36,539 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:36,539 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:36,540 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:36,540 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:36,540 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:36,540 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775025713 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775056532
2024-11-16T16:37:36,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39987:39987),(127.0.0.1/127.0.0.1:36347:36347)]
2024-11-16T16:37:36,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775025713 is not closed yet, will try archiving it next time
2024-11-16T16:37:36,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741833_1009 (size=5546)
2024-11-16T16:37:36,542 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:36,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741833_1009 (size=5546)
2024-11-16T16:37:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:36,546 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-16T16:37:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-16T16:37:36,547 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-16T16:37:36,547 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-16T16:37:36,686 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:36,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-16T16:37:36,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:36,701 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 64c8f3425b0e0c591a6235842242a340 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:37:36,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/ed36ca09cc064f92bcbf926961e432b3 is 1080, key is row0003/info:/1731775056529/Put/seqid=0
2024-11-16T16:37:36,717 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741840_1016 (size=6033)
2024-11-16T16:37:36,718 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741840_1016 (size=6033)
2024-11-16T16:37:36,718 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/ed36ca09cc064f92bcbf926961e432b3
2024-11-16T16:37:36,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/ed36ca09cc064f92bcbf926961e432b3 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3
2024-11-16T16:37:36,732 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3, entries=1, sequenceid=13, filesize=5.9 K
2024-11-16T16:37:36,733 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 32ms, sequenceid=13, compaction requested=true
2024-11-16T16:37:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 64c8f3425b0e0c591a6235842242a340:

2024-11-16T16:37:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:36,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-16T16:37:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-16T16:37:36,738 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-16T16:37:36,738 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 188 msec
2024-11-16T16:37:36,740 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 196 msec
2024-11-16T16:37:37,495 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:37,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:38,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:38,687 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:39,496 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:39,688 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:40,497 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:40,689 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:41,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:41,690 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:42,498 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:42,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:43,499 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:43,691 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:44,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:44,692 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:45,500 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:45,693 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:46,238 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-16T16:37:46,238 INFO  [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-16T16:37:46,501 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-16T16:37:46,558 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-16T16:37:46,558 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:37:46,560 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:37:46,560 DEBUG [Time-limited test {}] regionserver.HStore(1541): 64c8f3425b0e0c591a6235842242a340/info is initiating minor compaction (all files)
2024-11-16T16:37:46,560 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:37:46,560 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:37:46,560 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 64c8f3425b0e0c591a6235842242a340/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:46,561 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3] into tmpdir=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp, totalSize=17.7 K
2024-11-16T16:37:46,561 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 53479e6e3015444c9b06892f4924d567, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731775036366
2024-11-16T16:37:46,562 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f38bcc43375f4831a3c17521deffee01, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731775046458
2024-11-16T16:37:46,563 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ed36ca09cc064f92bcbf926961e432b3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731775056529
2024-11-16T16:37:46,577 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): 64c8f3425b0e0c591a6235842242a340#info#compaction#47 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:37:46,577 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/ca49dc692b654c7f9be0d0e3fb1310ff is 1080, key is row0001/info:/1731775036366/Put/seqid=0
2024-11-16T16:37:46,582 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741841_1017 (size=8296)
2024-11-16T16:37:46,583 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741841_1017 (size=8296)
2024-11-16T16:37:46,590 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/ca49dc692b654c7f9be0d0e3fb1310ff as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ca49dc692b654c7f9be0d0e3fb1310ff
2024-11-16T16:37:46,598 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 64c8f3425b0e0c591a6235842242a340/info of 64c8f3425b0e0c591a6235842242a340 into ca49dc692b654c7f9be0d0e3fb1310ff(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:37:46,598 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 64c8f3425b0e0c591a6235842242a340:

2024-11-16T16:37:46,601 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C42251%2C1731775025092.1731775066601
2024-11-16T16:37:46,608 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:46,608 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:46,608 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:46,608 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:46,608 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:46,608 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775056532 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775066601
2024-11-16T16:37:46,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39987:39987),(127.0.0.1/127.0.0.1:36347:36347)]
2024-11-16T16:37:46,610 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775056532 is not closed yet, will try archiving it next time
2024-11-16T16:37:46,610 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775025713 to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs/303fe01965e8%2C42251%2C1731775025092.1731775025713
2024-11-16T16:37:46,610 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741839_1015 (size=2520)
2024-11-16T16:37:46,611 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741839_1015 (size=2520)
2024-11-16T16:37:46,611 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:37:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-16T16:37:46,614 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-16T16:37:46,615 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-16T16:37:46,615 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-16T16:37:46,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:46,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42251 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-16T16:37:46,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:46,769 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 64c8f3425b0e0c591a6235842242a340 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:37:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/8c5aaeb5abb54d5c90b42aa404e87191 is 1080, key is row0000/info:/1731775066599/Put/seqid=0
2024-11-16T16:37:46,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741843_1019 (size=6033)
2024-11-16T16:37:46,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741843_1019 (size=6033)
2024-11-16T16:37:46,780 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/8c5aaeb5abb54d5c90b42aa404e87191
2024-11-16T16:37:46,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/8c5aaeb5abb54d5c90b42aa404e87191 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/8c5aaeb5abb54d5c90b42aa404e87191
2024-11-16T16:37:46,792 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/8c5aaeb5abb54d5c90b42aa404e87191, entries=1, sequenceid=18, filesize=5.9 K
2024-11-16T16:37:46,794 INFO  [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 25ms, sequenceid=18, compaction requested=false
2024-11-16T16:37:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 64c8f3425b0e0c591a6235842242a340:

2024-11-16T16:37:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:46,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-16T16:37:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-16T16:37:46,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-16T16:37:46,799 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 181 msec
2024-11-16T16:37:46,802 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 189 msec
2024-11-16T16:37:47,502 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:47,694 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:48,503 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:48,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:49,504 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:49,695 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:50,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:50,696 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:51,505 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:51,681 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 64c8f3425b0e0c591a6235842242a340, had cached 0 bytes from a total of 14329
2024-11-16T16:37:51,697 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:52,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:52,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:53,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:53,698 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:54,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:54,699 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:55,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:55,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:56,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-16T16:37:56,687 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-16T16:37:56,690 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C42251%2C1731775025092.1731775076689
2024-11-16T16:37:56,697 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,697 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,698 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,698 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,698 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,698 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775066601 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775076689
2024-11-16T16:37:56,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36347:36347),(127.0.0.1/127.0.0.1:39987:39987)]
2024-11-16T16:37:56,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775066601 is not closed yet, will try archiving it next time
2024-11-16T16:37:56,699 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/WALs/303fe01965e8,42251,1731775025092/303fe01965e8%2C42251%2C1731775025092.1731775056532 to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs/303fe01965e8%2C42251%2C1731775025092.1731775056532
2024-11-16T16:37:56,699 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:37:56,699 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:37:56,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:37:56,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:56,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:56,700 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:37:56,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:37:56,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741842_1018 (size=2026)
2024-11-16T16:37:56,700 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=424949578, stopped=false
2024-11-16T16:37:56,700 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,37995,1731775024920
2024-11-16T16:37:56,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741842_1018 (size=2026)
2024-11-16T16:37:56,700 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:56,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:37:56,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:37:56,775 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:37:56,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:56,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:56,775 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:37:56,776 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:37:56,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:56,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:37:56,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:37:56,776 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,42251,1731775025092' *****
2024-11-16T16:37:56,776 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:37:56,776 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(3091): Received CLOSE for 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:56,777 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,42251,1731775025092
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:37:56,777 INFO  [RS:0;303fe01965e8:42251 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:42251.
2024-11-16T16:37:56,777 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64c8f3425b0e0c591a6235842242a340, disabling compactions & flushes
2024-11-16T16:37:56,777 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:56,778 DEBUG [RS:0;303fe01965e8:42251 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:37:56,778 DEBUG [RS:0;303fe01965e8:42251 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:56,778 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:56,778 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340. after waiting 0 ms
2024-11-16T16:37:56,778 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:37:56,778 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:56,778 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:37:56,778 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:37:56,778 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:37:56,778 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 64c8f3425b0e0c591a6235842242a340 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-16T16:37:56,778 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-16T16:37:56,778 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 64c8f3425b0e0c591a6235842242a340=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.}
2024-11-16T16:37:56,778 DEBUG [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 64c8f3425b0e0c591a6235842242a340
2024-11-16T16:37:56,778 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:37:56,778 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:37:56,779 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:37:56,779 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:37:56,779 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:37:56,779 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-16T16:37:56,783 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/d6b9ac439a59414691927ff42e25f9fd is 1080, key is row0001/info:/1731775076688/Put/seqid=0
2024-11-16T16:37:56,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741845_1021 (size=6033)
2024-11-16T16:37:56,788 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741845_1021 (size=6033)
2024-11-16T16:37:56,789 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/d6b9ac439a59414691927ff42e25f9fd
2024-11-16T16:37:56,795 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/.tmp/info/d6b9ac439a59414691927ff42e25f9fd as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/d6b9ac439a59414691927ff42e25f9fd
2024-11-16T16:37:56,798 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/info/6ed797075e284e27a4d7de2635d842b0 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340./info:regioninfo/1731775026698/Put/seqid=0
2024-11-16T16:37:56,802 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/d6b9ac439a59414691927ff42e25f9fd, entries=1, sequenceid=22, filesize=5.9 K
2024-11-16T16:37:56,803 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 25ms, sequenceid=22, compaction requested=true
2024-11-16T16:37:56,807 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3] to archive
2024-11-16T16:37:56,808 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:37:56,809 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567 to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/53479e6e3015444c9b06892f4924d567
2024-11-16T16:37:56,811 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01 to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/f38bcc43375f4831a3c17521deffee01
2024-11-16T16:37:56,812 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3 to hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/info/ed36ca09cc064f92bcbf926961e432b3
2024-11-16T16:37:56,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741846_1022 (size=7308)
2024-11-16T16:37:56,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741846_1022 (size=7308)
2024-11-16T16:37:56,813 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=303fe01965e8:37995 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-16T16:37:56,813 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [53479e6e3015444c9b06892f4924d567=6033, f38bcc43375f4831a3c17521deffee01=6033, ed36ca09cc064f92bcbf926961e432b3=6033]
2024-11-16T16:37:56,816 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/info/6ed797075e284e27a4d7de2635d842b0
2024-11-16T16:37:56,817 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/64c8f3425b0e0c591a6235842242a340/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-16T16:37:56,818 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:56,818 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64c8f3425b0e0c591a6235842242a340:
Waiting for close lock at 1731775076777Running coprocessor pre-close hooks at 1731775076777Disabling compacts and flushes for region at 1731775076777Disabling writes for close at 1731775076778 (+1 ms)Obtaining lock to block concurrent updates at 1731775076778Preparing flush snapshotting stores in 64c8f3425b0e0c591a6235842242a340 at 1731775076778Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731775076778Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340. at 1731775076779 (+1 ms)Flushing 64c8f3425b0e0c591a6235842242a340/info: creating writer at 1731775076779Flushing 64c8f3425b0e0c591a6235842242a340/info: appending metadata at 1731775076783 (+4 ms)Flushing 64c8f3425b0e0c591a6235842242a340/info: closing flushed file at 1731775076783Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7544727e: reopening flushed file at 1731775076794 (+11 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 64c8f3425b0e0c591a6235842242a340 in 25ms, sequenceid=22, compaction requested=true at 1731775076803 (+9 ms)Writing region close event to WAL at 1731775076813 (+10 ms)Running coprocessor post-close hooks at 1731775076817 (+4 ms)Closed at 1731775076817
2024-11-16T16:37:56,818 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731775026332.64c8f3425b0e0c591a6235842242a340.
2024-11-16T16:37:56,837 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/ns/da2bf2b089fc42dfac8d79056f21547f is 43, key is default/ns:d/1731775026222/Put/seqid=0
2024-11-16T16:37:56,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741847_1023 (size=5153)
2024-11-16T16:37:56,842 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741847_1023 (size=5153)
2024-11-16T16:37:56,843 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/ns/da2bf2b089fc42dfac8d79056f21547f
2024-11-16T16:37:56,865 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/table/0e96a038fd014ecbbed68d975688fa3d is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731775026709/Put/seqid=0
2024-11-16T16:37:56,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741848_1024 (size=5508)
2024-11-16T16:37:56,870 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741848_1024 (size=5508)
2024-11-16T16:37:56,870 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/table/0e96a038fd014ecbbed68d975688fa3d
2024-11-16T16:37:56,877 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/info/6ed797075e284e27a4d7de2635d842b0 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/info/6ed797075e284e27a4d7de2635d842b0
2024-11-16T16:37:56,883 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/info/6ed797075e284e27a4d7de2635d842b0, entries=10, sequenceid=11, filesize=7.1 K
2024-11-16T16:37:56,884 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/ns/da2bf2b089fc42dfac8d79056f21547f as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/ns/da2bf2b089fc42dfac8d79056f21547f
2024-11-16T16:37:56,890 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/ns/da2bf2b089fc42dfac8d79056f21547f, entries=2, sequenceid=11, filesize=5.0 K
2024-11-16T16:37:56,891 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/.tmp/table/0e96a038fd014ecbbed68d975688fa3d as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/table/0e96a038fd014ecbbed68d975688fa3d
2024-11-16T16:37:56,898 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/table/0e96a038fd014ecbbed68d975688fa3d, entries=2, sequenceid=11, filesize=5.4 K
2024-11-16T16:37:56,899 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false
2024-11-16T16:37:56,904 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-16T16:37:56,904 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:37:56,904 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:37:56,904 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775076778Running coprocessor pre-close hooks at 1731775076778Disabling compacts and flushes for region at 1731775076778Disabling writes for close at 1731775076779 (+1 ms)Obtaining lock to block concurrent updates at 1731775076779Preparing flush snapshotting stores in 1588230740 at 1731775076779Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731775076779Flushing stores of hbase:meta,,1.1588230740 at 1731775076780 (+1 ms)Flushing 1588230740/info: creating writer at 1731775076780Flushing 1588230740/info: appending metadata at 1731775076798 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731775076798Flushing 1588230740/ns: creating writer at 1731775076822 (+24 ms)Flushing 1588230740/ns: appending metadata at 1731775076837 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731775076837Flushing 1588230740/table: creating writer at 1731775076848 (+11 ms)Flushing 1588230740/table: appending metadata at 1731775076865 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731775076865Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@541b9b45: reopening flushed file at 1731775076876 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d0439e7: reopening flushed file at 1731775076883 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8a1bf1: reopening flushed file at 1731775076890 (+7 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false at 1731775076899 (+9 ms)Writing region close event to WAL at 1731775076900 (+1 ms)Running coprocessor post-close hooks at 1731775076904 (+4 ms)Closed at 1731775076904
2024-11-16T16:37:56,905 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:37:56,979 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,42251,1731775025092; all regions closed.
2024-11-16T16:37:56,979 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,979 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,980 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,980 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,980 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,982 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741834_1010 (size=3306)
2024-11-16T16:37:56,983 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741834_1010 (size=3306)
2024-11-16T16:37:56,986 DEBUG [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs
2024-11-16T16:37:56,986 INFO  [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C42251%2C1731775025092.meta:.meta(num 1731775026137)
2024-11-16T16:37:56,987 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,987 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,987 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,987 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,987 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:56,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741844_1020 (size=1252)
2024-11-16T16:37:56,989 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741844_1020 (size=1252)
2024-11-16T16:37:56,994 DEBUG [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/oldWALs
2024-11-16T16:37:56,994 INFO  [RS:0;303fe01965e8:42251 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C42251%2C1731775025092:(num 1731775076689)
2024-11-16T16:37:56,994 DEBUG [RS:0;303fe01965e8:42251 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:37:56,994 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:37:56,994 INFO  [RS:0;303fe01965e8:42251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:37:56,994 INFO  [RS:0;303fe01965e8:42251 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-16T16:37:56,994 INFO  [RS:0;303fe01965e8:42251 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:37:56,994 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:37:56,995 INFO  [RS:0;303fe01965e8:42251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42251
2024-11-16T16:37:57,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,42251,1731775025092
2024-11-16T16:37:57,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:37:57,006 INFO  [RS:0;303fe01965e8:42251 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:37:57,007 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,42251,1731775025092]
2024-11-16T16:37:57,027 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,42251,1731775025092 already deleted, retry=false
2024-11-16T16:37:57,027 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,42251,1731775025092 expired; onlineServers=0
2024-11-16T16:37:57,027 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,37995,1731775024920' *****
2024-11-16T16:37:57,027 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:37:57,028 DEBUG [M:0;303fe01965e8:37995 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:37:57,028 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:37:57,028 DEBUG [M:0;303fe01965e8:37995 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:37:57,028 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775025434 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775025434,5,FailOnTimeoutGroup]
2024-11-16T16:37:57,028 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775025434 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775025434,5,FailOnTimeoutGroup]
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:37:57,028 DEBUG [M:0;303fe01965e8:37995 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:37:57,028 INFO  [M:0;303fe01965e8:37995 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:37:57,029 INFO  [M:0;303fe01965e8:37995 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:37:57,029 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:37:57,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:37:57,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:37:57,038 DEBUG [M:0;303fe01965e8:37995 {}] zookeeper.ZKUtil(347): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:37:57,038 WARN  [M:0;303fe01965e8:37995 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:37:57,039 INFO  [M:0;303fe01965e8:37995 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/.lastflushedseqids
2024-11-16T16:37:57,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741849_1025 (size=130)
2024-11-16T16:37:57,045 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741849_1025 (size=130)
2024-11-16T16:37:57,046 INFO  [M:0;303fe01965e8:37995 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:37:57,046 INFO  [M:0;303fe01965e8:37995 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:37:57,046 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:37:57,046 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:57,046 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:57,046 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:37:57,046 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:57,047 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.60 KB heapSize=55.01 KB
2024-11-16T16:37:57,067 DEBUG [M:0;303fe01965e8:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d83f364962342cb8203b286b0cf3c0c is 82, key is hbase:meta,,1/info:regioninfo/1731775026166/Put/seqid=0
2024-11-16T16:37:57,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741850_1026 (size=5672)
2024-11-16T16:37:57,078 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741850_1026 (size=5672)
2024-11-16T16:37:57,079 INFO  [M:0;303fe01965e8:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d83f364962342cb8203b286b0cf3c0c
2024-11-16T16:37:57,106 DEBUG [M:0;303fe01965e8:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57126e120964433b8621f9e20ec757e4 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731775026714/Put/seqid=0
2024-11-16T16:37:57,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741851_1027 (size=7824)
2024-11-16T16:37:57,111 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741851_1027 (size=7824)
2024-11-16T16:37:57,112 INFO  [M:0;303fe01965e8:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.00 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57126e120964433b8621f9e20ec757e4
2024-11-16T16:37:57,117 INFO  [M:0;303fe01965e8:37995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57126e120964433b8621f9e20ec757e4
2024-11-16T16:37:57,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:57,117 INFO  [RS:0;303fe01965e8:42251 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:37:57,117 INFO  [RS:0;303fe01965e8:42251 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,42251,1731775025092; zookeeper connection closed.
2024-11-16T16:37:57,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42251-0x1014481cab30001, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:57,118 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12f7248f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12f7248f
2024-11-16T16:37:57,118 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:37:57,137 DEBUG [M:0;303fe01965e8:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8d7f3f36bdd4aabbc8e074246ce6b31 is 69, key is 303fe01965e8,42251,1731775025092/rs:state/1731775025556/Put/seqid=0
2024-11-16T16:37:57,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741852_1028 (size=5156)
2024-11-16T16:37:57,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741852_1028 (size=5156)
2024-11-16T16:37:57,144 INFO  [M:0;303fe01965e8:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8d7f3f36bdd4aabbc8e074246ce6b31
2024-11-16T16:37:57,171 DEBUG [M:0;303fe01965e8:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d781a5a611a4c73a3bca377aa3ace00 is 52, key is load_balancer_on/state:d/1731775026327/Put/seqid=0
2024-11-16T16:37:57,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741853_1029 (size=5056)
2024-11-16T16:37:57,176 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741853_1029 (size=5056)
2024-11-16T16:37:57,177 INFO  [M:0;303fe01965e8:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d781a5a611a4c73a3bca377aa3ace00
2024-11-16T16:37:57,183 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d83f364962342cb8203b286b0cf3c0c as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d83f364962342cb8203b286b0cf3c0c
2024-11-16T16:37:57,188 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d83f364962342cb8203b286b0cf3c0c, entries=8, sequenceid=121, filesize=5.5 K
2024-11-16T16:37:57,189 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57126e120964433b8621f9e20ec757e4 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57126e120964433b8621f9e20ec757e4
2024-11-16T16:37:57,194 INFO  [M:0;303fe01965e8:37995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57126e120964433b8621f9e20ec757e4
2024-11-16T16:37:57,194 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57126e120964433b8621f9e20ec757e4, entries=14, sequenceid=121, filesize=7.6 K
2024-11-16T16:37:57,195 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a8d7f3f36bdd4aabbc8e074246ce6b31 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a8d7f3f36bdd4aabbc8e074246ce6b31
2024-11-16T16:37:57,201 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a8d7f3f36bdd4aabbc8e074246ce6b31, entries=1, sequenceid=121, filesize=5.0 K
2024-11-16T16:37:57,202 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4d781a5a611a4c73a3bca377aa3ace00 as hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4d781a5a611a4c73a3bca377aa3ace00
2024-11-16T16:37:57,207 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37859/user/jenkins/test-data/9aae2512-a5e8-3b35-2e2b-ea5757061794/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4d781a5a611a4c73a3bca377aa3ace00, entries=1, sequenceid=121, filesize=4.9 K
2024-11-16T16:37:57,208 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=121, compaction requested=false
2024-11-16T16:37:57,210 INFO  [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:37:57,210 DEBUG [M:0;303fe01965e8:37995 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775077046Disabling compacts and flushes for region at 1731775077046Disabling writes for close at 1731775077046Obtaining lock to block concurrent updates at 1731775077047 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731775077047Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44647, getHeapSize=56264, getOffHeapSize=0, getCellsCount=140 at 1731775077047Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731775077048 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731775077048Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731775077066 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731775077067 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731775077084 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731775077105 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731775077105Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731775077117 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731775077137 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731775077137Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731775077150 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731775077170 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731775077170Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65642956: reopening flushed file at 1731775077182 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@183092c0: reopening flushed file at 1731775077188 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4832d848: reopening flushed file at 1731775077195 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79952a0d: reopening flushed file at 1731775077201 (+6 ms)Finished flush of dataSize ~43.60 KB/44647, heapSize ~54.95 KB/56264, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 162ms, sequenceid=121, compaction requested=false at 1731775077208 (+7 ms)Writing region close event to WAL at 1731775077210 (+2 ms)Closed at 1731775077210
2024-11-16T16:37:57,210 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:57,210 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:57,210 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:57,210 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:57,210 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:37:57,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35931 is added to blk_1073741830_1006 (size=53044)
2024-11-16T16:37:57,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45701 is added to blk_1073741830_1006 (size=53044)
2024-11-16T16:37:57,213 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:37:57,213 INFO  [M:0;303fe01965e8:37995 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:37:57,213 INFO  [M:0;303fe01965e8:37995 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37995
2024-11-16T16:37:57,213 INFO  [M:0;303fe01965e8:37995 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:37:57,321 INFO  [M:0;303fe01965e8:37995 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:37:57,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:57,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x1014481cab30000, quorum=127.0.0.1:64939, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:37:57,324 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3d01a0e4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:57,324 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166de790{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:57,324 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:57,324 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acc16de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:57,324 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33307a55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:57,326 WARN  [BP-1894627294-172.17.0.2-1731775022339 heartbeating to localhost/127.0.0.1:37859 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:37:57,326 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:37:57,326 WARN  [BP-1894627294-172.17.0.2-1731775022339 heartbeating to localhost/127.0.0.1:37859 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1894627294-172.17.0.2-1731775022339 (Datanode Uuid c799d081-c560-4e53-b335-73d91d53dd56) service to localhost/127.0.0.1:37859
2024-11-16T16:37:57,326 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:37:57,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data3/current/BP-1894627294-172.17.0.2-1731775022339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:57,327 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data4/current/BP-1894627294-172.17.0.2-1731775022339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:57,327 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:37:57,330 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3272a875{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:57,330 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@61918bd4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:57,330 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:57,330 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40a21084{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:57,331 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46bd2b1a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:57,332 WARN  [BP-1894627294-172.17.0.2-1731775022339 heartbeating to localhost/127.0.0.1:37859 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:37:57,332 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:37:57,332 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:37:57,332 WARN  [BP-1894627294-172.17.0.2-1731775022339 heartbeating to localhost/127.0.0.1:37859 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1894627294-172.17.0.2-1731775022339 (Datanode Uuid 99dcb861-7903-49d5-8b0b-3c91babd2180) service to localhost/127.0.0.1:37859
2024-11-16T16:37:57,333 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data1/current/BP-1894627294-172.17.0.2-1731775022339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:57,333 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/cluster_499271e2-e5e4-f371-da5f-0fe75560dd76/data/data2/current/BP-1894627294-172.17.0.2-1731775022339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:37:57,333 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:37:57,340 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6694026d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:37:57,341 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2bd70390{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:37:57,341 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:37:57,341 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75c50826{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:37:57,341 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a3a01a6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir/,STOPPED}
2024-11-16T16:37:57,346 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:37:57,369 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:37:57,377 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=209 (was 183)
Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37859
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37859 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37859 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37859
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37859
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37859
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:37859 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:37859
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/303fe01965e8:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37859
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=176 (was 207), ProcessCount=11 (was 11), AvailableMemoryMB=3462 (was 3554)
2024-11-16T16:37:57,385 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=209, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=176, ProcessCount=11, AvailableMemoryMB=3462
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.log.dir so I do NOT create it in target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5370049-a96a-6201-29e4-9bb5148c152d/hadoop.tmp.dir so I do NOT create it in target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0, deleteOnExit=true
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/test.cache.data in system properties and HBase conf
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:37:57,386 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:37:57,387 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:37:57,387 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:37:57,388 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:37:57,403 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:37:57,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:57,579 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:37:57,701 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:57,814 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:57,818 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:57,819 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:57,819 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:57,819 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:37:57,819 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:57,820 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6150e164{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:57,820 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ec6f4ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:57,922 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5350cad{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/java.io.tmpdir/jetty-localhost-44389-hadoop-hdfs-3_4_1-tests_jar-_-any-1188806554601931480/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:37:57,922 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@278f505{HTTP/1.1, (http/1.1)}{localhost:44389}
2024-11-16T16:37:57,922 INFO  [Time-limited test {}] server.Server(415): Started @253698ms
2024-11-16T16:37:57,935 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:37:58,191 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:58,194 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:58,195 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:58,195 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:58,195 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:37:58,195 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43cde2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:58,196 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13fa93c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:58,305 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2079ccf2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/java.io.tmpdir/jetty-localhost-33365-hadoop-hdfs-3_4_1-tests_jar-_-any-12186045690337652966/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:58,305 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13da1053{HTTP/1.1, (http/1.1)}{localhost:33365}
2024-11-16T16:37:58,305 INFO  [Time-limited test {}] server.Server(415): Started @254081ms
2024-11-16T16:37:58,306 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:37:58,344 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:37:58,347 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:37:58,348 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:37:58,348 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:37:58,348 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:37:58,348 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5169cded{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:37:58,348 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75eba18f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:37:58,461 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea981d3{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/java.io.tmpdir/jetty-localhost-40461-hadoop-hdfs-3_4_1-tests_jar-_-any-3172549969762339401/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:37:58,461 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71c2dc3e{HTTP/1.1, (http/1.1)}{localhost:40461}
2024-11-16T16:37:58,461 INFO  [Time-limited test {}] server.Server(415): Started @254237ms
2024-11-16T16:37:58,462 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:37:58,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:58,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:59,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:59,702 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:37:59,789 WARN  [Thread-1984 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data1/current/BP-192065853-172.17.0.2-1731775077407/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:59,789 WARN  [Thread-1985 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data2/current/BP-192065853-172.17.0.2-1731775077407/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:59,815 WARN  [Thread-1948 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:37:59,817 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf70141a2c0495e97 with lease ID 0xfb6e4167a29393de: Processing first storage report for DS-58f6e2b4-d866-4178-a140-3d7b25b78d28 from datanode DatanodeRegistration(127.0.0.1:43367, datanodeUuid=a80ad598-d00f-4457-bc9a-92d3bf96e6b7, infoPort=34751, infoSecurePort=0, ipcPort=45663, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407)
2024-11-16T16:37:59,817 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf70141a2c0495e97 with lease ID 0xfb6e4167a29393de: from storage DS-58f6e2b4-d866-4178-a140-3d7b25b78d28 node DatanodeRegistration(127.0.0.1:43367, datanodeUuid=a80ad598-d00f-4457-bc9a-92d3bf96e6b7, infoPort=34751, infoSecurePort=0, ipcPort=45663, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:37:59,817 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf70141a2c0495e97 with lease ID 0xfb6e4167a29393de: Processing first storage report for DS-344e8dd4-0e89-4bd2-aa6d-aebd05554221 from datanode DatanodeRegistration(127.0.0.1:43367, datanodeUuid=a80ad598-d00f-4457-bc9a-92d3bf96e6b7, infoPort=34751, infoSecurePort=0, ipcPort=45663, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407)
2024-11-16T16:37:59,817 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf70141a2c0495e97 with lease ID 0xfb6e4167a29393de: from storage DS-344e8dd4-0e89-4bd2-aa6d-aebd05554221 node DatanodeRegistration(127.0.0.1:43367, datanodeUuid=a80ad598-d00f-4457-bc9a-92d3bf96e6b7, infoPort=34751, infoSecurePort=0, ipcPort=45663, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:59,954 WARN  [Thread-1995 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data3/current/BP-192065853-172.17.0.2-1731775077407/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:59,954 WARN  [Thread-1996 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data4/current/BP-192065853-172.17.0.2-1731775077407/current, will proceed with Du for space computation calculation, 
2024-11-16T16:37:59,971 WARN  [Thread-1971 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:37:59,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x667dc5de2a0fb0a9 with lease ID 0xfb6e4167a29393df: Processing first storage report for DS-c4d4d667-0f07-40ae-83e4-1442192b389a from datanode DatanodeRegistration(127.0.0.1:45481, datanodeUuid=9bbdc873-5fd2-4959-bed6-9f7b0296d9ab, infoPort=36855, infoSecurePort=0, ipcPort=43675, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407)
2024-11-16T16:37:59,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x667dc5de2a0fb0a9 with lease ID 0xfb6e4167a29393df: from storage DS-c4d4d667-0f07-40ae-83e4-1442192b389a node DatanodeRegistration(127.0.0.1:45481, datanodeUuid=9bbdc873-5fd2-4959-bed6-9f7b0296d9ab, infoPort=36855, infoSecurePort=0, ipcPort=43675, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:37:59,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x667dc5de2a0fb0a9 with lease ID 0xfb6e4167a29393df: Processing first storage report for DS-12e15ae3-fd60-405e-9861-2ba4bdc9f49d from datanode DatanodeRegistration(127.0.0.1:45481, datanodeUuid=9bbdc873-5fd2-4959-bed6-9f7b0296d9ab, infoPort=36855, infoSecurePort=0, ipcPort=43675, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407)
2024-11-16T16:37:59,973 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x667dc5de2a0fb0a9 with lease ID 0xfb6e4167a29393df: from storage DS-12e15ae3-fd60-405e-9861-2ba4bdc9f49d node DatanodeRegistration(127.0.0.1:45481, datanodeUuid=9bbdc873-5fd2-4959-bed6-9f7b0296d9ab, infoPort=36855, infoSecurePort=0, ipcPort=43675, storageInfo=lv=-57;cid=testClusterID;nsid=283392586;c=1731775077407), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-16T16:37:59,998 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05
2024-11-16T16:38:00,019 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/zookeeper_0, clientPort=65180, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:38:00,020 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65180
2024-11-16T16:38:00,021 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,022 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:38:00,032 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:38:00,033 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba with version=8
2024-11-16T16:38:00,033 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,035 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:38:00,036 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:38:00,036 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:38:00,037 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35141
2024-11-16T16:38:00,039 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35141 connecting to ZooKeeper ensemble=127.0.0.1:65180
2024-11-16T16:38:00,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351410x0, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:38:00,106 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35141-0x1014482a1fd0000 connected
2024-11-16T16:38:00,196 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,198 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:00,201 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba, hbase.cluster.distributed=false
2024-11-16T16:38:00,206 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:38:00,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35141
2024-11-16T16:38:00,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35141
2024-11-16T16:38:00,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35141
2024-11-16T16:38:00,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35141
2024-11-16T16:38:00,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35141
2024-11-16T16:38:00,237 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:38:00,237 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,237 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,237 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:38:00,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:00,238 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:38:00,238 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:38:00,238 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:38:00,239 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36337
2024-11-16T16:38:00,241 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36337 connecting to ZooKeeper ensemble=127.0.0.1:65180
2024-11-16T16:38:00,242 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,244 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,259 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363370x0, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:38:00,260 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:363370x0, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:00,260 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:38:00,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36337-0x1014482a1fd0001 connected
2024-11-16T16:38:00,263 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:38:00,264 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:38:00,265 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:38:00,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36337
2024-11-16T16:38:00,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36337
2024-11-16T16:38:00,267 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36337
2024-11-16T16:38:00,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36337
2024-11-16T16:38:00,268 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36337
2024-11-16T16:38:00,279 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:35141
2024-11-16T16:38:00,280 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:00,290 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:00,291 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,301 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:38:00,301 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,301 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:38:00,302 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,35141,1731775080035 from backup master directory
2024-11-16T16:38:00,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,311 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:00,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:00,311 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:38:00,311 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,315 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/hbase.id] with ID: 9385dd77-b953-407a-a1d8-8535c73108f6
2024-11-16T16:38:00,315 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/.tmp/hbase.id
2024-11-16T16:38:00,328 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:38:00,329 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:38:00,329 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/.tmp/hbase.id]:[hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/hbase.id]
2024-11-16T16:38:00,342 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:00,342 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:38:00,344 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-16T16:38:00,353 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,381 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:38:00,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:38:00,382 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:38:00,383 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:38:00,384 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:00,393 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:38:00,394 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:38:00,401 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store
2024-11-16T16:38:00,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:38:00,408 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:38:00,409 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:00,409 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:00,409 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775080409Disabling compacts and flushes for region at 1731775080409Disabling writes for close at 1731775080409Writing region close event to WAL at 1731775080409Closed at 1731775080409
2024-11-16T16:38:00,410 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/.initializing
2024-11-16T16:38:00,410 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/WALs/303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,413 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C35141%2C1731775080035, suffix=, logDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/WALs/303fe01965e8,35141,1731775080035, archiveDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/oldWALs, maxLogs=10
2024-11-16T16:38:00,413 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C35141%2C1731775080035.1731775080413
2024-11-16T16:38:00,419 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/WALs/303fe01965e8,35141,1731775080035/303fe01965e8%2C35141%2C1731775080035.1731775080413
2024-11-16T16:38:00,419 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36855:36855),(127.0.0.1/127.0.0.1:34751:34751)]
2024-11-16T16:38:00,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:38:00,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:00,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,420 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,426 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:00,428 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:38:00,428 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,429 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:00,429 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:00,430 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:38:00,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,430 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:00,430 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:00,431 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:38:00,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,432 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:00,432 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:00,433 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:38:00,433 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,434 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:00,434 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,435 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,435 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,437 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,437 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,437 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:38:00,438 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:00,440 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:38:00,441 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=781771, jitterRate=-0.005927920341491699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:38:00,441 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731775080420Initializing all the Stores at 1731775080421 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775080421Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775080426 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775080426Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775080426Cleaning up temporary data from old regions at 1731775080437 (+11 ms)Region opened successfully at 1731775080441 (+4 ms)
2024-11-16T16:38:00,442 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:38:00,445 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1465362a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:38:00,446 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:38:00,446 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:38:00,446 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:38:00,446 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:38:00,447 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:38:00,447 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:38:00,447 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:38:00,449 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:38:00,450 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:38:00,459 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:38:00,459 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:38:00,460 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:38:00,469 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:38:00,470 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:38:00,470 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:38:00,480 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:38:00,481 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:38:00,490 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:38:00,492 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:38:00,505 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:38:00,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:00,515 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:00,515 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:00,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,516 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,35141,1731775080035, sessionid=0x1014482a1fd0000, setting cluster-up flag (Was=false)
2024-11-16T16:38:00,536 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,568 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:38:00,569 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,589 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:00,621 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:38:00,622 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,35141,1731775080035
2024-11-16T16:38:00,623 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:38:00,625 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:00,625 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:38:00,625 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:38:00,625 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,35141,1731775080035
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:38:00,626 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:38:00,627 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731775110630
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:38:00,630 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:38:00,631 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,631 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:00,631 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:38:00,631 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:38:00,631 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:38:00,631 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:38:00,631 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:38:00,632 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:38:00,632 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775080632,5,FailOnTimeoutGroup]
2024-11-16T16:38:00,632 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775080632,5,FailOnTimeoutGroup]
2024-11-16T16:38:00,632 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,632 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:38:00,632 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,632 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,632 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,632 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:38:00,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:38:00,639 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:38:00,640 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:38:00,640 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba
2024-11-16T16:38:00,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:38:00,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:38:00,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:00,648 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:38:00,649 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:38:00,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,650 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:00,650 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:38:00,651 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:38:00,651 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,651 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:00,651 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:38:00,652 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:38:00,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:00,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:38:00,654 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:38:00,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:00,654 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:00,654 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:38:00,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740
2024-11-16T16:38:00,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740
2024-11-16T16:38:00,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:38:00,657 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:38:00,657 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:38:00,659 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:38:00,662 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:38:00,662 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=809560, jitterRate=0.029409483075141907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731775080647Initializing all the Stores at 1731775080647Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775080647Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775080648 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775080648Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775080648Cleaning up temporary data from old regions at 1731775080657 (+9 ms)Region opened successfully at 1731775080663 (+6 ms)
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:38:00,663 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:38:00,663 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:00,663 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775080663Disabling compacts and flushes for region at 1731775080663Disabling writes for close at 1731775080663Writing region close event to WAL at 1731775080663Closed at 1731775080663
2024-11-16T16:38:00,664 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:00,664 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:38:00,664 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:38:00,665 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:38:00,666 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:38:00,670 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(746): ClusterId : 9385dd77-b953-407a-a1d8-8535c73108f6
2024-11-16T16:38:00,670 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:38:00,681 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:38:00,681 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:38:00,691 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:38:00,691 DEBUG [RS:0;303fe01965e8:36337 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33356877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:38:00,703 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:00,708 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:36337
2024-11-16T16:38:00,708 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:38:00,708 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:38:00,708 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:38:00,709 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,35141,1731775080035 with port=36337, startcode=1731775080237
2024-11-16T16:38:00,709 DEBUG [RS:0;303fe01965e8:36337 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:38:00,711 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48299, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:38:00,711 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35141 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,711 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35141 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,713 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba
2024-11-16T16:38:00,713 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34573
2024-11-16T16:38:00,713 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:38:00,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:38:00,722 DEBUG [RS:0;303fe01965e8:36337 {}] zookeeper.ZKUtil(111): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,722 WARN  [RS:0;303fe01965e8:36337 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:38:00,722 INFO  [RS:0;303fe01965e8:36337 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:00,722 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,723 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,36337,1731775080237]
2024-11-16T16:38:00,726 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:38:00,727 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:38:00,728 INFO  [RS:0;303fe01965e8:36337 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:38:00,728 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,728 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:38:00,729 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:38:00,729 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:38:00,729 DEBUG [RS:0;303fe01965e8:36337 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,730 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,36337,1731775080237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:38:00,746 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:38:00,746 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,36337,1731775080237-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,746 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,747 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.Replication(171): 303fe01965e8,36337,1731775080237 started
2024-11-16T16:38:00,762 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:00,762 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,36337,1731775080237, RpcServer on 303fe01965e8/172.17.0.2:36337, sessionid=0x1014482a1fd0001
2024-11-16T16:38:00,763 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:38:00,763 DEBUG [RS:0;303fe01965e8:36337 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,763 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,36337,1731775080237'
2024-11-16T16:38:00,763 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:38:00,763 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,36337,1731775080237
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,36337,1731775080237'
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:38:00,764 DEBUG [RS:0;303fe01965e8:36337 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:38:00,764 INFO  [RS:0;303fe01965e8:36337 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:38:00,764 INFO  [RS:0;303fe01965e8:36337 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:38:00,817 WARN  [303fe01965e8:35141 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:38:00,866 INFO  [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C36337%2C1731775080237, suffix=, logDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237, archiveDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs, maxLogs=32
2024-11-16T16:38:00,867 INFO  [RS:0;303fe01965e8:36337 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36337%2C1731775080237.1731775080867
2024-11-16T16:38:00,872 INFO  [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775080867
2024-11-16T16:38:00,876 DEBUG [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34751:34751),(127.0.0.1/127.0.0.1:36855:36855)]
2024-11-16T16:38:01,067 DEBUG [303fe01965e8:35141 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:38:01,067 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:01,069 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,36337,1731775080237, state=OPENING
2024-11-16T16:38:01,124 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:38:01,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:01,136 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:01,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:01,137 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:01,137 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:38:01,137 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,36337,1731775080237}]
2024-11-16T16:38:01,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:38:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:38:01,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-16T16:38:01,290 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:38:01,292 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38747, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:38:01,296 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:38:01,296 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:01,298 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C36337%2C1731775080237.meta, suffix=.meta, logDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237, archiveDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs, maxLogs=32
2024-11-16T16:38:01,298 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36337%2C1731775080237.meta.1731775081298.meta
2024-11-16T16:38:01,303 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.meta.1731775081298.meta
2024-11-16T16:38:01,311 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34751:34751),(127.0.0.1/127.0.0.1:36855:36855)]
2024-11-16T16:38:01,312 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:38:01,312 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:38:01,312 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:38:01,312 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:38:01,312 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:38:01,312 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:01,313 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:38:01,313 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:38:01,314 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:38:01,314 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:38:01,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,315 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:01,315 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:38:01,316 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:38:01,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,316 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:01,316 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:38:01,317 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:38:01,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,317 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:01,317 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:38:01,318 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:38:01,318 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,319 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:01,319 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:38:01,320 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740
2024-11-16T16:38:01,321 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740
2024-11-16T16:38:01,322 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:38:01,322 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:38:01,322 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:38:01,324 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:38:01,325 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=865786, jitterRate=0.10090391337871552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:38:01,325 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:38:01,326 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731775081313Writing region info on filesystem at 1731775081313Initializing all the Stores at 1731775081313Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775081313Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775081314 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775081314Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775081314Cleaning up temporary data from old regions at 1731775081322 (+8 ms)Running coprocessor post-open hooks at 1731775081325 (+3 ms)Region opened successfully at 1731775081325
2024-11-16T16:38:01,326 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731775081289
2024-11-16T16:38:01,329 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:38:01,329 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:38:01,330 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:01,331 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,36337,1731775080237, state=OPEN
2024-11-16T16:38:01,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:38:01,375 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:38:01,376 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,36337,1731775080237
2024-11-16T16:38:01,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:01,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:01,379 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:38:01,379 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,36337,1731775080237 in 239 msec
2024-11-16T16:38:01,382 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:38:01,382 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 715 msec
2024-11-16T16:38:01,383 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:01,383 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:38:01,385 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:38:01,385 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,36337,1731775080237, seqNum=-1]
2024-11-16T16:38:01,385 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:38:01,387 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:38:01,394 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 768 msec
2024-11-16T16:38:01,394 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731775081394, completionTime=-1
2024-11-16T16:38:01,394 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:38:01,394 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731775141397
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775201397
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:35141, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,397 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,398 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,400 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.091sec
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:38:01,403 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:38:01,404 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:38:01,406 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:38:01,406 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:38:01,406 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,35141,1731775080035-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:01,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4122179, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:01,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,35141,-1 for getting cluster id
2024-11-16T16:38:01,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:38:01,471 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9385dd77-b953-407a-a1d8-8535c73108f6'
2024-11-16T16:38:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:38:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9385dd77-b953-407a-a1d8-8535c73108f6"

2024-11-16T16:38:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5733ce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,35141,-1]
2024-11-16T16:38:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:38:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:01,473 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:38:01,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d25570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:01,474 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:38:01,475 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,36337,1731775080237, seqNum=-1]
2024-11-16T16:38:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:38:01,477 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:38:01,479 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,35141,1731775080035
2024-11-16T16:38:01,479 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:01,482 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:38:01,483 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-16T16:38:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,35141,1731775080035
2024-11-16T16:38:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7504be1d
2024-11-16T16:38:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-16T16:38:01,485 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-16T16:38:01,486 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-16T16:38:01,486 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-16T16:38:01,486 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:38:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-16T16:38:01,489 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-16T16:38:01,489 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,489 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-16T16:38:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:38:01,490 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-16T16:38:01,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741835_1011 (size=381)
2024-11-16T16:38:01,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741835_1011 (size=381)
2024-11-16T16:38:01,507 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7529d189d7c11f7fb002d1d2a22cbda8, NAME => 'TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba
2024-11-16T16:38:01,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:01,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741836_1012 (size=64)
2024-11-16T16:38:01,514 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741836_1012 (size=64)
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 7529d189d7c11f7fb002d1d2a22cbda8, disabling compactions & flushes
2024-11-16T16:38:01,515 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. after waiting 0 ms
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,515 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,515 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7529d189d7c11f7fb002d1d2a22cbda8:
Waiting for close lock at 1731775081515Disabling compacts and flushes for region at 1731775081515Disabling writes for close at 1731775081515Writing region close event to WAL at 1731775081515Closed at 1731775081515
2024-11-16T16:38:01,516 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-16T16:38:01,517 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731775081516"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731775081516"}]},"ts":"1731775081516"}
2024-11-16T16:38:01,519 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-16T16:38:01,521 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-16T16:38:01,521 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731775081521"}]},"ts":"1731775081521"}
2024-11-16T16:38:01,523 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-16T16:38:01,523 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, ASSIGN}]
2024-11-16T16:38:01,525 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, ASSIGN
2024-11-16T16:38:01,526 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, ASSIGN; state=OFFLINE, location=303fe01965e8,36337,1731775080237; forceNewPlan=false, retain=false
2024-11-16T16:38:01,677 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7529d189d7c11f7fb002d1d2a22cbda8, regionState=OPENING, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:01,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, ASSIGN because future has completed
2024-11-16T16:38:01,680 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237}]
2024-11-16T16:38:01,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:01,818 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,819 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,820 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,837 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7529d189d7c11f7fb002d1d2a22cbda8, NAME => 'TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:38:01,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:01,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,839 INFO  [StoreOpener-7529d189d7c11f7fb002d1d2a22cbda8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 7529d189d7c11f7fb002d1d2a22cbda8 
2024-11-16T16:38:01,840 INFO  [StoreOpener-7529d189d7c11f7fb002d1d2a22cbda8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7529d189d7c11f7fb002d1d2a22cbda8 columnFamilyName info
2024-11-16T16:38:01,840 DEBUG [StoreOpener-7529d189d7c11f7fb002d1d2a22cbda8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:01,841 INFO  [StoreOpener-7529d189d7c11f7fb002d1d2a22cbda8-1 {}] regionserver.HStore(327): Store=7529d189d7c11f7fb002d1d2a22cbda8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:01,841 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,841 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,842 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,842 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,842 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,843 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,844 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,844 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,844 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,844 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,846 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:38:01,846 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7529d189d7c11f7fb002d1d2a22cbda8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=717706, jitterRate=-0.0873899757862091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:38:01,846 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:01,847 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7529d189d7c11f7fb002d1d2a22cbda8:
Running coprocessor pre-open hook at 1731775081837Writing region info on filesystem at 1731775081837Initializing all the Stores at 1731775081838 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775081838Cleaning up temporary data from old regions at 1731775081842 (+4 ms)Running coprocessor post-open hooks at 1731775081846 (+4 ms)Region opened successfully at 1731775081847 (+1 ms)
2024-11-16T16:38:01,848 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., pid=6, masterSystemTime=1731775081833
2024-11-16T16:38:01,848 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,849 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,849 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,850 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,850 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:01,851 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7529d189d7c11f7fb002d1d2a22cbda8, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:01,851 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:01,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237 because future has completed
2024-11-16T16:38:01,857 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-16T16:38:01,857 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237 in 176 msec
2024-11-16T16:38:01,860 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-16T16:38:01,860 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, ASSIGN in 334 msec
2024-11-16T16:38:01,861 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-16T16:38:01,861 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731775081861"}]},"ts":"1731775081861"}
2024-11-16T16:38:01,863 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-16T16:38:01,865 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-16T16:38:01,867 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 379 msec
2024-11-16T16:38:02,357 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:38:02,358 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,358 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,358 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,358 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,359 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,360 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,388 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,389 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,390 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,396 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,399 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:02,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:02,704 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:03,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:03,705 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:04,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:04,706 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:05,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:05,707 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:06,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:06,708 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:06,726 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:38:06,728 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-16T16:38:07,342 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:38:07,343 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,343 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,385 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,385 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,385 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,386 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,392 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,393 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,397 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:07,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:07,709 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:08,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:08,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:09,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:09,710 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:10,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:10,711 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:11,188 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:38:11,189 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-16T16:38:11,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:38:11,189 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-16T16:38:11,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-16T16:38:11,189 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-16T16:38:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35141 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-16T16:38:11,507 INFO  [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-16T16:38:11,507 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-16T16:38:11,510 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-16T16:38:11,510 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:11,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., hostname=303fe01965e8,36337,1731775080237, seqNum=2]
2024-11-16T16:38:11,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:11,527 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:38:11,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e1da70692d314106b819863bdf45ed66 is 1080, key is row0001/info:/1731775091513/Put/seqid=0
2024-11-16T16:38:11,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741837_1013 (size=12509)
2024-11-16T16:38:11,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741837_1013 (size=12509)
2024-11-16T16:38:11,550 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e1da70692d314106b819863bdf45ed66
2024-11-16T16:38:11,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e1da70692d314106b819863bdf45ed66 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66
2024-11-16T16:38:11,563 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66, entries=7, sequenceid=11, filesize=12.2 K
2024-11-16T16:38:11,564 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=16.81 KB/17216 for 7529d189d7c11f7fb002d1d2a22cbda8 in 37ms, sequenceid=11, compaction requested=false
2024-11-16T16:38:11,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:11,565 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB
2024-11-16T16:38:11,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/8ba1587ce3254b34ac4107a71cdab6ee is 1080, key is row0008/info:/1731775091528/Put/seqid=0
2024-11-16T16:38:11,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741838_1014 (size=23299)
2024-11-16T16:38:11,578 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741838_1014 (size=23299)
2024-11-16T16:38:11,578 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/8ba1587ce3254b34ac4107a71cdab6ee
2024-11-16T16:38:11,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/8ba1587ce3254b34ac4107a71cdab6ee as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee
2024-11-16T16:38:11,588 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee, entries=17, sequenceid=31, filesize=22.8 K
2024-11-16T16:38:11,589 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=8.41 KB/8608 for 7529d189d7c11f7fb002d1d2a22cbda8 in 24ms, sequenceid=31, compaction requested=false
2024-11-16T16:38:11,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:11,590 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=35.0 K, sizeToCheck=16.0 K
2024-11-16T16:38:11,590 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:11,590 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee because midkey is the same as first or last row
2024-11-16T16:38:11,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:12,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:12,712 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:13,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:13,584 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB
2024-11-16T16:38:13,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/400913ee130249a292d9a6579ab05793 is 1080, key is row0025/info:/1731775091566/Put/seqid=0
2024-11-16T16:38:13,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741839_1015 (size=14663)
2024-11-16T16:38:13,601 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741839_1015 (size=14663)
2024-11-16T16:38:13,601 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=43 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/400913ee130249a292d9a6579ab05793
2024-11-16T16:38:13,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/400913ee130249a292d9a6579ab05793 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793
2024-11-16T16:38:13,613 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793, entries=9, sequenceid=43, filesize=14.3 K
2024-11-16T16:38:13,614 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=13.66 KB/13988 for 7529d189d7c11f7fb002d1d2a22cbda8 in 29ms, sequenceid=43, compaction requested=true
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=49.3 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee because midkey is the same as first or last row
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7529d189d7c11f7fb002d1d2a22cbda8:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:13,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:13,614 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-16T16:38:13,614 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:13,615 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 50471 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:13,615 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 7529d189d7c11f7fb002d1d2a22cbda8/info is initiating minor compaction (all files)
2024-11-16T16:38:13,615 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7529d189d7c11f7fb002d1d2a22cbda8/info in TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:13,616 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp, totalSize=49.3 K
2024-11-16T16:38:13,616 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1da70692d314106b819863bdf45ed66, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731775091513
2024-11-16T16:38:13,616 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba1587ce3254b34ac4107a71cdab6ee, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731775091528
2024-11-16T16:38:13,617 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400913ee130249a292d9a6579ab05793, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731775091566
2024-11-16T16:38:13,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/83b57de1772c453db4f481d2626cace0 is 1080, key is row0034/info:/1731775093586/Put/seqid=0
2024-11-16T16:38:13,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741840_1016 (size=20064)
2024-11-16T16:38:13,625 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741840_1016 (size=20064)
2024-11-16T16:38:13,626 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/83b57de1772c453db4f481d2626cace0
2024-11-16T16:38:13,632 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7529d189d7c11f7fb002d1d2a22cbda8#info#compaction#61 average throughput is 16.93 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:13,632 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/93a806506cdf4460b6f6a502217f1e60 is 1080, key is row0001/info:/1731775091513/Put/seqid=0
2024-11-16T16:38:13,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/83b57de1772c453db4f481d2626cace0 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0
2024-11-16T16:38:13,639 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0, entries=14, sequenceid=60, filesize=19.6 K
2024-11-16T16:38:13,640 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 7529d189d7c11f7fb002d1d2a22cbda8 in 26ms, sequenceid=60, compaction requested=false
2024-11-16T16:38:13,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:13,640 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=68.9 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,640 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,640 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee because midkey is the same as first or last row
2024-11-16T16:38:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:13,641 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:13,641 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741841_1017 (size=40670)
2024-11-16T16:38:13,642 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741841_1017 (size=40670)
2024-11-16T16:38:13,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/2e0cba154c7f43f5a6d33e893f1d5a8c is 1080, key is row0048/info:/1731775093615/Put/seqid=0
2024-11-16T16:38:13,651 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/93a806506cdf4460b6f6a502217f1e60 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60
2024-11-16T16:38:13,658 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7529d189d7c11f7fb002d1d2a22cbda8/info of 7529d189d7c11f7fb002d1d2a22cbda8 into 93a806506cdf4460b6f6a502217f1e60(size=39.7 K), total size for store is 59.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:13,658 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., storeName=7529d189d7c11f7fb002d1d2a22cbda8/info, priority=13, startTime=1731775093614; duration=0sec
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60 because midkey is the same as first or last row
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,658 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60 because midkey is the same as first or last row
2024-11-16T16:38:13,659 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=59.3 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,659 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,659 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60 because midkey is the same as first or last row
2024-11-16T16:38:13,659 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:13,659 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7529d189d7c11f7fb002d1d2a22cbda8:info
2024-11-16T16:38:13,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741842_1018 (size=17894)
2024-11-16T16:38:13,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741842_1018 (size=17894)
2024-11-16T16:38:13,662 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/2e0cba154c7f43f5a6d33e893f1d5a8c
2024-11-16T16:38:13,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/2e0cba154c7f43f5a6d33e893f1d5a8c as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c
2024-11-16T16:38:13,676 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c, entries=12, sequenceid=75, filesize=17.5 K
2024-11-16T16:38:13,677 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=5.25 KB/5380 for 7529d189d7c11f7fb002d1d2a22cbda8 in 36ms, sequenceid=75, compaction requested=true
2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=76.8 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60 because midkey is the same as first or last row
2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7529d189d7c11f7fb002d1d2a22cbda8:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:13,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:13,678 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:13,679 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 78628 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:13,679 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 7529d189d7c11f7fb002d1d2a22cbda8/info is initiating minor compaction (all files)
2024-11-16T16:38:13,680 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7529d189d7c11f7fb002d1d2a22cbda8/info in TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:13,680 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp, totalSize=76.8 K
2024-11-16T16:38:13,680 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a806506cdf4460b6f6a502217f1e60, keycount=33, bloomtype=ROW, size=39.7 K, encoding=NONE, compression=NONE, seqNum=43, earliestPutTs=1731775091513
2024-11-16T16:38:13,680 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83b57de1772c453db4f481d2626cace0, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731775093586
2024-11-16T16:38:13,681 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e0cba154c7f43f5a6d33e893f1d5a8c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731775093615
2024-11-16T16:38:13,695 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7529d189d7c11f7fb002d1d2a22cbda8#info#compaction#63 average throughput is 20.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:13,696 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/69d76e714bb14ad19a45f51a21a156a8 is 1080, key is row0001/info:/1731775091513/Put/seqid=0
2024-11-16T16:38:13,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741843_1019 (size=68843)
2024-11-16T16:38:13,707 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741843_1019 (size=68843)
2024-11-16T16:38:13,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:13,714 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/69d76e714bb14ad19a45f51a21a156a8 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8
2024-11-16T16:38:13,722 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7529d189d7c11f7fb002d1d2a22cbda8/info of 7529d189d7c11f7fb002d1d2a22cbda8 into 69d76e714bb14ad19a45f51a21a156a8(size=67.2 K), total size for store is 67.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:13,722 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., storeName=7529d189d7c11f7fb002d1d2a22cbda8/info, priority=13, startTime=1731775093678; duration=0sec
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.2 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.2 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,722 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:13,723 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=67.2 K, sizeToCheck=16.0 K
2024-11-16T16:38:13,723 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:13,723 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:13,723 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:13,723 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7529d189d7c11f7fb002d1d2a22cbda8:info
2024-11-16T16:38:14,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:14,713 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:15,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:15,658 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:38:15,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/736e5ac02c9340228a79e35c21464169 is 1080, key is row0060/info:/1731775093643/Put/seqid=0
2024-11-16T16:38:15,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741844_1020 (size=12509)
2024-11-16T16:38:15,669 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741844_1020 (size=12509)
2024-11-16T16:38:15,669 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/736e5ac02c9340228a79e35c21464169
2024-11-16T16:38:15,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/736e5ac02c9340228a79e35c21464169 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169
2024-11-16T16:38:15,682 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169, entries=7, sequenceid=87, filesize=12.2 K
2024-11-16T16:38:15,683 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=10.51 KB/10760 for 7529d189d7c11f7fb002d1d2a22cbda8 in 25ms, sequenceid=87, compaction requested=false
2024-11-16T16:38:15,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:15,683 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=79.4 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,683 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,683 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:15,684 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-16T16:38:15,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/10ccc64cd31840cc8ccce076025aeb15 is 1080, key is row0067/info:/1731775095659/Put/seqid=0
2024-11-16T16:38:15,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741845_1021 (size=16817)
2024-11-16T16:38:15,697 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741845_1021 (size=16817)
2024-11-16T16:38:15,698 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/10ccc64cd31840cc8ccce076025aeb15
2024-11-16T16:38:15,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/10ccc64cd31840cc8ccce076025aeb15 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15
2024-11-16T16:38:15,713 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15, entries=11, sequenceid=101, filesize=16.4 K
2024-11-16T16:38:15,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:15,714 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for 7529d189d7c11f7fb002d1d2a22cbda8 in 30ms, sequenceid=101, compaction requested=true
2024-11-16T16:38:15,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:15,715 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=95.9 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,715 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,715 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:15,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7529d189d7c11f7fb002d1d2a22cbda8:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:15,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:15,715 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:15,716 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 98169 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:15,716 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 7529d189d7c11f7fb002d1d2a22cbda8/info is initiating minor compaction (all files)
2024-11-16T16:38:15,716 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7529d189d7c11f7fb002d1d2a22cbda8/info in TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:15,717 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp, totalSize=95.9 K
2024-11-16T16:38:15,717 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69d76e714bb14ad19a45f51a21a156a8, keycount=59, bloomtype=ROW, size=67.2 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731775091513
2024-11-16T16:38:15,717 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 736e5ac02c9340228a79e35c21464169, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731775093643
2024-11-16T16:38:15,718 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ccc64cd31840cc8ccce076025aeb15, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731775095659
2024-11-16T16:38:15,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:15,718 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-16T16:38:15,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/cbee1cd6927f47b78a68a9ca9ce0fb20 is 1080, key is row0078/info:/1731775095685/Put/seqid=0
2024-11-16T16:38:15,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741846_1022 (size=20064)
2024-11-16T16:38:15,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741846_1022 (size=20064)
2024-11-16T16:38:15,733 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/cbee1cd6927f47b78a68a9ca9ce0fb20
2024-11-16T16:38:15,733 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7529d189d7c11f7fb002d1d2a22cbda8#info#compaction#67 average throughput is 26.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:15,734 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/3bbcf780bb1347c29591a44dfd7f4bb0 is 1080, key is row0001/info:/1731775091513/Put/seqid=0
2024-11-16T16:38:15,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741847_1023 (size=88408)
2024-11-16T16:38:15,738 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741847_1023 (size=88408)
2024-11-16T16:38:15,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/cbee1cd6927f47b78a68a9ca9ce0fb20 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/cbee1cd6927f47b78a68a9ca9ce0fb20
2024-11-16T16:38:15,745 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/3bbcf780bb1347c29591a44dfd7f4bb0 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0
2024-11-16T16:38:15,746 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/cbee1cd6927f47b78a68a9ca9ce0fb20, entries=14, sequenceid=118, filesize=19.6 K
2024-11-16T16:38:15,747 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=5.25 KB/5380 for 7529d189d7c11f7fb002d1d2a22cbda8 in 28ms, sequenceid=118, compaction requested=false
2024-11-16T16:38:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=115.5 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 because midkey is the same as first or last row
2024-11-16T16:38:15,751 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7529d189d7c11f7fb002d1d2a22cbda8/info of 7529d189d7c11f7fb002d1d2a22cbda8 into 3bbcf780bb1347c29591a44dfd7f4bb0(size=86.3 K), total size for store is 105.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:15,751 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7529d189d7c11f7fb002d1d2a22cbda8:

2024-11-16T16:38:15,752 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., storeName=7529d189d7c11f7fb002d1d2a22cbda8/info, priority=13, startTime=1731775095715; duration=0sec
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K
2024-11-16T16:38:15,752 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-16T16:38:15,754 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:15,754 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:15,754 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7529d189d7c11f7fb002d1d2a22cbda8:info
2024-11-16T16:38:15,756 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35141 {}] assignment.AssignmentManager(1355): Split request from 303fe01965e8,36337,1731775080237, parent={ENCODED => 7529d189d7c11f7fb002d1d2a22cbda8, NAME => 'TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-16T16:38:15,762 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35141 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=303fe01965e8,36337,1731775080237
2024-11-16T16:38:15,767 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35141 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=7529d189d7c11f7fb002d1d2a22cbda8, daughterA=ccab8207ddcd64af7f8730b3b8374c0d, daughterB=5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:15,768 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=7529d189d7c11f7fb002d1d2a22cbda8, daughterA=ccab8207ddcd64af7f8730b3b8374c0d, daughterB=5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:15,768 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=7529d189d7c11f7fb002d1d2a22cbda8, daughterA=ccab8207ddcd64af7f8730b3b8374c0d, daughterB=5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:15,768 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=7529d189d7c11f7fb002d1d2a22cbda8, daughterA=ccab8207ddcd64af7f8730b3b8374c0d, daughterB=5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:15,775 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, UNASSIGN}]
2024-11-16T16:38:15,777 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, UNASSIGN
2024-11-16T16:38:15,779 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7529d189d7c11f7fb002d1d2a22cbda8, regionState=CLOSING, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:15,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, UNASSIGN because future has completed
2024-11-16T16:38:15,783 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-16T16:38:15,783 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237}]
2024-11-16T16:38:15,941 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:15,941 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-16T16:38:15,942 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 7529d189d7c11f7fb002d1d2a22cbda8, disabling compactions & flushes
2024-11-16T16:38:15,942 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:15,942 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:15,942 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. after waiting 0 ms
2024-11-16T16:38:15,942 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:15,942 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 7529d189d7c11f7fb002d1d2a22cbda8 1/1 column families, dataSize=5.25 KB heapSize=5.88 KB
2024-11-16T16:38:15,946 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e33a136ecaff44339f1abff751face67 is 1080, key is row0092/info:/1731775095720/Put/seqid=0
2024-11-16T16:38:15,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741848_1024 (size=10347)
2024-11-16T16:38:15,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741848_1024 (size=10347)
2024-11-16T16:38:15,952 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.25 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e33a136ecaff44339f1abff751face67
2024-11-16T16:38:15,956 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/.tmp/info/e33a136ecaff44339f1abff751face67 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e33a136ecaff44339f1abff751face67
2024-11-16T16:38:15,961 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e33a136ecaff44339f1abff751face67, entries=5, sequenceid=127, filesize=10.1 K
2024-11-16T16:38:15,962 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 7529d189d7c11f7fb002d1d2a22cbda8 in 20ms, sequenceid=127, compaction requested=true
2024-11-16T16:38:15,964 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15] to archive
2024-11-16T16:38:15,965 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:38:15,967 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e1da70692d314106b819863bdf45ed66
2024-11-16T16:38:15,968 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/8ba1587ce3254b34ac4107a71cdab6ee
2024-11-16T16:38:15,969 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/93a806506cdf4460b6f6a502217f1e60
2024-11-16T16:38:15,971 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/400913ee130249a292d9a6579ab05793
2024-11-16T16:38:15,972 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/83b57de1772c453db4f481d2626cace0
2024-11-16T16:38:15,973 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/69d76e714bb14ad19a45f51a21a156a8
2024-11-16T16:38:15,975 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/2e0cba154c7f43f5a6d33e893f1d5a8c
2024-11-16T16:38:15,978 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/736e5ac02c9340228a79e35c21464169
2024-11-16T16:38:15,979 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/10ccc64cd31840cc8ccce076025aeb15
2024-11-16T16:38:15,985 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=1
2024-11-16T16:38:15,986 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.
2024-11-16T16:38:15,986 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 7529d189d7c11f7fb002d1d2a22cbda8:
Waiting for close lock at 1731775095942Running coprocessor pre-close hooks at 1731775095942Disabling compacts and flushes for region at 1731775095942Disabling writes for close at 1731775095942Obtaining lock to block concurrent updates at 1731775095942Preparing flush snapshotting stores in 7529d189d7c11f7fb002d1d2a22cbda8 at 1731775095942Finished memstore snapshotting TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., syncing WAL and waiting on mvcc, flushsize=dataSize=5380, getHeapSize=6000, getOffHeapSize=0, getCellsCount=5 at 1731775095942Flushing stores of TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. at 1731775095943 (+1 ms)Flushing 7529d189d7c11f7fb002d1d2a22cbda8/info: creating writer at 1731775095943Flushing 7529d189d7c11f7fb002d1d2a22cbda8/info: appending metadata at 1731775095945 (+2 ms)Flushing 7529d189d7c11f7fb002d1d2a22cbda8/info: closing flushed file at 1731775095945Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77fffa7e: reopening flushed file at 1731775095956 (+11 ms)Finished flush of dataSize ~5.25 KB/5380, heapSize ~5.86 KB/6000, currentSize=0 B/0 for 7529d189d7c11f7fb002d1d2a22cbda8 in 20ms, sequenceid=127, compaction requested=true at 1731775095962 (+6 ms)Writing region close event to WAL at 1731775095982 (+20 ms)Running coprocessor post-close hooks at 1731775095986 (+4 ms)Closed at 1731775095986
2024-11-16T16:38:15,989 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:15,989 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7529d189d7c11f7fb002d1d2a22cbda8, regionState=CLOSED
2024-11-16T16:38:15,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237 because future has completed
2024-11-16T16:38:15,996 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-16T16:38:15,996 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 7529d189d7c11f7fb002d1d2a22cbda8, server=303fe01965e8,36337,1731775080237 in 210 msec
2024-11-16T16:38:15,998 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-16T16:38:15,998 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=7529d189d7c11f7fb002d1d2a22cbda8, UNASSIGN in 221 msec
2024-11-16T16:38:16,013 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:16,017 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=7529d189d7c11f7fb002d1d2a22cbda8, threads=3
2024-11-16T16:38:16,019 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,019 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/cbee1cd6927f47b78a68a9ca9ce0fb20 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,020 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e33a136ecaff44339f1abff751face67 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,041 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/cbee1cd6927f47b78a68a9ca9ce0fb20, top=true
2024-11-16T16:38:16,052 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e33a136ecaff44339f1abff751face67, top=true
2024-11-16T16:38:16,066 INFO  [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20 for child: 5ca511962c567b6c405af457006a4a3e, parent: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,067 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/cbee1cd6927f47b78a68a9ca9ce0fb20 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,081 INFO  [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67 for child: 5ca511962c567b6c405af457006a4a3e, parent: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,081 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/e33a136ecaff44339f1abff751face67 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,085 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741849_1025 (size=27)
2024-11-16T16:38:16,085 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741849_1025 (size=27)
2024-11-16T16:38:16,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741850_1026 (size=27)
2024-11-16T16:38:16,099 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741850_1026 (size=27)
2024-11-16T16:38:16,099 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0 for region: 7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:16,101 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 7529d189d7c11f7fb002d1d2a22cbda8 Daughter A: [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8] storefiles, Daughter B: [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67] storefiles.
2024-11-16T16:38:16,115 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741851_1027 (size=71)
2024-11-16T16:38:16,116 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741851_1027 (size=71)
2024-11-16T16:38:16,118 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:16,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741852_1028 (size=71)
2024-11-16T16:38:16,138 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741852_1028 (size=71)
2024-11-16T16:38:16,140 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:16,148 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-16T16:38:16,151 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/recovered.edits/130.seqid, newMaxSeqId=130, maxSeqId=-1
2024-11-16T16:38:16,153 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731775096152"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731775096152"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731775096152"}]},"ts":"1731775096152"}
2024-11-16T16:38:16,153 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731775096152"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731775096152"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731775096152"}]},"ts":"1731775096152"}
2024-11-16T16:38:16,153 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731775096152"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731775096152"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731775096152"}]},"ts":"1731775096152"}
2024-11-16T16:38:16,169 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ccab8207ddcd64af7f8730b3b8374c0d, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5ca511962c567b6c405af457006a4a3e, ASSIGN}]
2024-11-16T16:38:16,171 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5ca511962c567b6c405af457006a4a3e, ASSIGN
2024-11-16T16:38:16,171 INFO  [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ccab8207ddcd64af7f8730b3b8374c0d, ASSIGN
2024-11-16T16:38:16,172 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5ca511962c567b6c405af457006a4a3e, ASSIGN; state=SPLITTING_NEW, location=303fe01965e8,36337,1731775080237; forceNewPlan=false, retain=false
2024-11-16T16:38:16,172 INFO  [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ccab8207ddcd64af7f8730b3b8374c0d, ASSIGN; state=SPLITTING_NEW, location=303fe01965e8,36337,1731775080237; forceNewPlan=false, retain=false
2024-11-16T16:38:16,322 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ccab8207ddcd64af7f8730b3b8374c0d, regionState=OPENING, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:16,322 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5ca511962c567b6c405af457006a4a3e, regionState=OPENING, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:16,324 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ccab8207ddcd64af7f8730b3b8374c0d, ASSIGN because future has completed
2024-11-16T16:38:16,325 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccab8207ddcd64af7f8730b3b8374c0d, server=303fe01965e8,36337,1731775080237}]
2024-11-16T16:38:16,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5ca511962c567b6c405af457006a4a3e, ASSIGN because future has completed
2024-11-16T16:38:16,326 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ca511962c567b6c405af457006a4a3e, server=303fe01965e8,36337,1731775080237}]
2024-11-16T16:38:16,483 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:16,484 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 5ca511962c567b6c405af457006a4a3e, NAME => 'TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-16T16:38:16,484 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,484 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:16,485 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,485 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,487 INFO  [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 5ca511962c567b6c405af457006a4a3e 
2024-11-16T16:38:16,488 INFO  [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ca511962c567b6c405af457006a4a3e columnFamilyName info
2024-11-16T16:38:16,489 DEBUG [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:16,503 DEBUG [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-top
2024-11-16T16:38:16,508 DEBUG [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20
2024-11-16T16:38:16,511 DEBUG [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67
2024-11-16T16:38:16,511 INFO  [StoreOpener-5ca511962c567b6c405af457006a4a3e-1 {}] regionserver.HStore(327): Store=5ca511962c567b6c405af457006a4a3e/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:16,511 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,512 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,513 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,513 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,513 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,515 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,516 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 5ca511962c567b6c405af457006a4a3e; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=705342, jitterRate=-0.10311159491539001}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:38:16,516 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:16,516 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 5ca511962c567b6c405af457006a4a3e:
Running coprocessor pre-open hook at 1731775096485Writing region info on filesystem at 1731775096485Initializing all the Stores at 1731775096486 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775096487 (+1 ms)Cleaning up temporary data from old regions at 1731775096513 (+26 ms)Running coprocessor post-open hooks at 1731775096516 (+3 ms)Region opened successfully at 1731775096516
2024-11-16T16:38:16,517 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., pid=13, masterSystemTime=1731775096477
2024-11-16T16:38:16,517 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:16,517 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:16,517 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:16,518 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:16,518 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:16,518 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:16,519 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-top, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=116.0 K
2024-11-16T16:38:16,519 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:16,519 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:16,519 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:16,520 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ccab8207ddcd64af7f8730b3b8374c0d, NAME => 'TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-16T16:38:16,520 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731775091513
2024-11-16T16:38:16,520 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,520 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:16,520 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=5ca511962c567b6c405af457006a4a3e, regionState=OPEN, openSeqNum=131, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:16,520 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,520 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,520 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731775095685
2024-11-16T16:38:16,521 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731775095720
2024-11-16T16:38:16,522 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-16T16:38:16,522 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-16T16:38:16,523 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-16T16:38:16,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5ca511962c567b6c405af457006a4a3e, server=303fe01965e8,36337,1731775080237 because future has completed
2024-11-16T16:38:16,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:16,527 INFO  [StoreOpener-ccab8207ddcd64af7f8730b3b8374c0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ccab8207ddcd64af7f8730b3b8374c0d 
2024-11-16T16:38:16,527 INFO  [StoreOpener-ccab8207ddcd64af7f8730b3b8374c0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ccab8207ddcd64af7f8730b3b8374c0d columnFamilyName info
2024-11-16T16:38:16,527 DEBUG [StoreOpener-ccab8207ddcd64af7f8730b3b8374c0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:16,528 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11
2024-11-16T16:38:16,528 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 5ca511962c567b6c405af457006a4a3e, server=303fe01965e8,36337,1731775080237 in 199 msec
2024-11-16T16:38:16,531 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=5ca511962c567b6c405af457006a4a3e, ASSIGN in 359 msec
2024-11-16T16:38:16,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/e083215a54fb4fa99048f4139799fef7 is 193, key is TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e./info:regioninfo/1731775096520/Put/seqid=0
2024-11-16T16:38:16,546 DEBUG [StoreOpener-ccab8207ddcd64af7f8730b3b8374c0d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-bottom
2024-11-16T16:38:16,546 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#70 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:16,546 INFO  [StoreOpener-ccab8207ddcd64af7f8730b3b8374c0d-1 {}] regionserver.HStore(327): Store=ccab8207ddcd64af7f8730b3b8374c0d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:16,546 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,547 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7468a952e0de4b3c93eb34a7ee834213 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:16,547 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,548 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,549 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,549 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741853_1029 (size=9882)
2024-11-16T16:38:16,550 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741853_1029 (size=9882)
2024-11-16T16:38:16,550 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/e083215a54fb4fa99048f4139799fef7
2024-11-16T16:38:16,551 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,551 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened ccab8207ddcd64af7f8730b3b8374c0d; next sequenceid=131; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=700193, jitterRate=-0.10965877771377563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-16T16:38:16,552 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:16,552 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ccab8207ddcd64af7f8730b3b8374c0d:
Running coprocessor pre-open hook at 1731775096520Writing region info on filesystem at 1731775096520Initializing all the Stores at 1731775096521 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775096521Cleaning up temporary data from old regions at 1731775096549 (+28 ms)Running coprocessor post-open hooks at 1731775096552 (+3 ms)Region opened successfully at 1731775096552
2024-11-16T16:38:16,553 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d., pid=12, masterSystemTime=1731775096477
2024-11-16T16:38:16,553 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store ccab8207ddcd64af7f8730b3b8374c0d:info, priority=-2147483648, current under compaction store size is 2
2024-11-16T16:38:16,553 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:16,553 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-16T16:38:16,554 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:16,554 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1541): ccab8207ddcd64af7f8730b3b8374c0d/info is initiating minor compaction (all files)
2024-11-16T16:38:16,554 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ccab8207ddcd64af7f8730b3b8374c0d/info in TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:16,554 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-bottom] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/.tmp, totalSize=86.3 K
2024-11-16T16:38:16,555 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8, keycount=38, bloomtype=ROW, size=86.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1731775091513
2024-11-16T16:38:16,555 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:16,555 INFO  [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:16,556 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ccab8207ddcd64af7f8730b3b8374c0d, regionState=OPEN, openSeqNum=131, regionLocation=303fe01965e8,36337,1731775080237
2024-11-16T16:38:16,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ccab8207ddcd64af7f8730b3b8374c0d, server=303fe01965e8,36337,1731775080237 because future has completed
2024-11-16T16:38:16,569 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741854_1030 (size=42984)
2024-11-16T16:38:16,570 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741854_1030 (size=42984)
2024-11-16T16:38:16,572 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10
2024-11-16T16:38:16,572 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ccab8207ddcd64af7f8730b3b8374c0d, server=303fe01965e8,36337,1731775080237 in 243 msec
2024-11-16T16:38:16,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-16T16:38:16,575 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=ccab8207ddcd64af7f8730b3b8374c0d, ASSIGN in 403 msec
2024-11-16T16:38:16,579 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=7529d189d7c11f7fb002d1d2a22cbda8, daughterA=ccab8207ddcd64af7f8730b3b8374c0d, daughterB=5ca511962c567b6c405af457006a4a3e in 813 msec
2024-11-16T16:38:16,579 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7468a952e0de4b3c93eb34a7ee834213 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7468a952e0de4b3c93eb34a7ee834213
2024-11-16T16:38:16,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/ns/d664ab38c21b45eeadec43391c33401e is 43, key is default/ns:d/1731775081387/Put/seqid=0
2024-11-16T16:38:16,586 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ccab8207ddcd64af7f8730b3b8374c0d#info#compaction#72 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:16,587 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/.tmp/info/32e27b0088f342b290097873b41777e3 is 1080, key is row0001/info:/1731775091513/Put/seqid=0
2024-11-16T16:38:16,589 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 7468a952e0de4b3c93eb34a7ee834213(size=42.0 K), total size for store is 42.0 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:16,589 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:16,589 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775096517; duration=0sec
2024-11-16T16:38:16,589 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:16,589 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:16,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741855_1031 (size=5153)
2024-11-16T16:38:16,598 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741855_1031 (size=5153)
2024-11-16T16:38:16,599 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/ns/d664ab38c21b45eeadec43391c33401e
2024-11-16T16:38:16,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741856_1032 (size=70862)
2024-11-16T16:38:16,609 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741856_1032 (size=70862)
2024-11-16T16:38:16,618 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/.tmp/info/32e27b0088f342b290097873b41777e3 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/32e27b0088f342b290097873b41777e3
2024-11-16T16:38:16,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/table/62321fd7ef754b58891f83c86bf4a1a6 is 65, key is TestLogRolling-testLogRolling/table:state/1731775081861/Put/seqid=0
2024-11-16T16:38:16,625 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in ccab8207ddcd64af7f8730b3b8374c0d/info of ccab8207ddcd64af7f8730b3b8374c0d into 32e27b0088f342b290097873b41777e3(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:16,625 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ccab8207ddcd64af7f8730b3b8374c0d:

2024-11-16T16:38:16,625 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d., storeName=ccab8207ddcd64af7f8730b3b8374c0d/info, priority=15, startTime=1731775096553; duration=0sec
2024-11-16T16:38:16,625 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:16,625 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ccab8207ddcd64af7f8730b3b8374c0d:info
2024-11-16T16:38:16,633 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741857_1033 (size=5340)
2024-11-16T16:38:16,634 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741857_1033 (size=5340)
2024-11-16T16:38:16,634 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/table/62321fd7ef754b58891f83c86bf4a1a6
2024-11-16T16:38:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/e083215a54fb4fa99048f4139799fef7 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/info/e083215a54fb4fa99048f4139799fef7
2024-11-16T16:38:16,648 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/info/e083215a54fb4fa99048f4139799fef7, entries=30, sequenceid=17, filesize=9.7 K
2024-11-16T16:38:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/ns/d664ab38c21b45eeadec43391c33401e as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/ns/d664ab38c21b45eeadec43391c33401e
2024-11-16T16:38:16,656 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/ns/d664ab38c21b45eeadec43391c33401e, entries=2, sequenceid=17, filesize=5.0 K
2024-11-16T16:38:16,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/table/62321fd7ef754b58891f83c86bf4a1a6 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/table/62321fd7ef754b58891f83c86bf4a1a6
2024-11-16T16:38:16,662 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/table/62321fd7ef754b58891f83c86bf4a1a6, entries=2, sequenceid=17, filesize=5.2 K
2024-11-16T16:38:16,663 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 141ms, sequenceid=17, compaction requested=false
2024-11-16T16:38:16,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-16T16:38:16,714 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:17,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:17,715 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:17,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:36140 deadline: 1731775107740, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. is not online on 303fe01965e8,36337,1731775080237
2024-11-16T16:38:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., hostname=303fe01965e8,36337,1731775080237, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., hostname=303fe01965e8,36337,1731775080237, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. is not online on 303fe01965e8,36337,1731775080237
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-16T16:38:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., hostname=303fe01965e8,36337,1731775080237, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8. is not online on 303fe01965e8,36337,1731775080237
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-16T16:38:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731775081486.7529d189d7c11f7fb002d1d2a22cbda8., hostname=303fe01965e8,36337,1731775080237, seqNum=2 from cache
2024-11-16T16:38:18,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:18,716 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:19,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:19,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:20,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:20,717 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:20,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,988 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,989 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:20,990 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,028 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,029 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,034 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,035 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,038 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:21,549 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,550 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,551 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,588 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,589 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,595 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,596 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,599 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:21,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:22,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:22,718 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:23,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:23,719 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:24,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:24,720 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:25,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:25,721 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:26,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:26,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:27,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:27,722 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:27,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., hostname=303fe01965e8,36337,1731775080237, seqNum=131]
2024-11-16T16:38:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:27,887 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:38:27,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/e5f700046e0b4bacaae18496dffeace4 is 1080, key is row0097/info:/1731775107879/Put/seqid=0
2024-11-16T16:38:27,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741858_1034 (size=12516)
2024-11-16T16:38:27,896 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741858_1034 (size=12516)
2024-11-16T16:38:27,897 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/e5f700046e0b4bacaae18496dffeace4
2024-11-16T16:38:27,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/e5f700046e0b4bacaae18496dffeace4 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4
2024-11-16T16:38:27,907 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4, entries=7, sequenceid=141, filesize=12.2 K
2024-11-16T16:38:27,908 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 5ca511962c567b6c405af457006a4a3e in 21ms, sequenceid=141, compaction requested=false
2024-11-16T16:38:27,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:27,909 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:27,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/72496ae50fb64175a2c2f13aeb7ccdd2 is 1080, key is row0104/info:/1731775107888/Put/seqid=0
2024-11-16T16:38:27,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741859_1035 (size=17906)
2024-11-16T16:38:27,923 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741859_1035 (size=17906)
2024-11-16T16:38:27,923 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/72496ae50fb64175a2c2f13aeb7ccdd2
2024-11-16T16:38:27,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/72496ae50fb64175a2c2f13aeb7ccdd2 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2
2024-11-16T16:38:27,933 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2, entries=12, sequenceid=156, filesize=17.5 K
2024-11-16T16:38:27,934 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=13.66 KB/13988 for 5ca511962c567b6c405af457006a4a3e in 25ms, sequenceid=156, compaction requested=true
2024-11-16T16:38:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:27,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:27,934 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:27,935 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 73406 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:27,935 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:27,935 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:27,935 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7468a952e0de4b3c93eb34a7ee834213, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=71.7 K
2024-11-16T16:38:27,935 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7468a952e0de4b3c93eb34a7ee834213, keycount=35, bloomtype=ROW, size=42.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731775093648
2024-11-16T16:38:27,936 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5f700046e0b4bacaae18496dffeace4, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731775107879
2024-11-16T16:38:27,936 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72496ae50fb64175a2c2f13aeb7ccdd2, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731775107888
2024-11-16T16:38:27,945 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#76 average throughput is 55.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:27,946 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a39d34d03bc645d1b0b6d6108c02b3c3 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:27,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741860_1036 (size=63636)
2024-11-16T16:38:27,951 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741860_1036 (size=63636)
2024-11-16T16:38:28,365 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a39d34d03bc645d1b0b6d6108c02b3c3 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a39d34d03bc645d1b0b6d6108c02b3c3
2024-11-16T16:38:28,372 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into a39d34d03bc645d1b0b6d6108c02b3c3(size=62.1 K), total size for store is 62.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:28,372 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:28,372 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775107934; duration=0sec
2024-11-16T16:38:28,372 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:28,372 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:28,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:28,723 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:29,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:29,724 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:29,937 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-16T16:38:29,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7c8e6671aa0c450fbbbb954792933f09 is 1080, key is row0116/info:/1731775107910/Put/seqid=0
2024-11-16T16:38:29,952 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741861_1037 (size=20078)
2024-11-16T16:38:29,953 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741861_1037 (size=20078)
2024-11-16T16:38:29,953 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7c8e6671aa0c450fbbbb954792933f09
2024-11-16T16:38:29,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7c8e6671aa0c450fbbbb954792933f09 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09
2024-11-16T16:38:29,963 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09, entries=14, sequenceid=174, filesize=19.6 K
2024-11-16T16:38:29,964 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=9.46 KB/9684 for 5ca511962c567b6c405af457006a4a3e in 27ms, sequenceid=174, compaction requested=false
2024-11-16T16:38:29,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:29,966 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-16T16:38:29,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/199ab26e663d4b549a9829e136eff1ee is 1080, key is row0130/info:/1731775109939/Put/seqid=0
2024-11-16T16:38:29,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741862_1038 (size=16828)
2024-11-16T16:38:29,973 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741862_1038 (size=16828)
2024-11-16T16:38:29,974 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/199ab26e663d4b549a9829e136eff1ee
2024-11-16T16:38:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/199ab26e663d4b549a9829e136eff1ee as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee
2024-11-16T16:38:29,985 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee, entries=11, sequenceid=188, filesize=16.4 K
2024-11-16T16:38:29,985 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=10.51 KB/10760 for 5ca511962c567b6c405af457006a4a3e in 19ms, sequenceid=188, compaction requested=true
2024-11-16T16:38:29,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:29,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:29,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:29,986 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:29,986 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-16T16:38:29,987 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 100542 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:29,987 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:29,987 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:29,987 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a39d34d03bc645d1b0b6d6108c02b3c3, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=98.2 K
2024-11-16T16:38:29,987 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39d34d03bc645d1b0b6d6108c02b3c3, keycount=54, bloomtype=ROW, size=62.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731775093648
2024-11-16T16:38:29,988 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c8e6671aa0c450fbbbb954792933f09, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731775107910
2024-11-16T16:38:29,988 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 199ab26e663d4b549a9829e136eff1ee, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731775109939
2024-11-16T16:38:29,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/dada926439204103b0e6d821a381abdf is 1080, key is row0141/info:/1731775109967/Put/seqid=0
2024-11-16T16:38:29,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741863_1039 (size=16828)
2024-11-16T16:38:29,998 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-16T16:38:29,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741863_1039 (size=16828)
2024-11-16T16:38:29,999 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/dada926439204103b0e6d821a381abdf
2024-11-16T16:38:30,002 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#80 average throughput is 40.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:30,003 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/43a91eb3dae84c8bb6b2ab71240c16a2 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:30,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/dada926439204103b0e6d821a381abdf as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf
2024-11-16T16:38:30,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741864_1040 (size=90765)
2024-11-16T16:38:30,007 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741864_1040 (size=90765)
2024-11-16T16:38:30,011 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf, entries=11, sequenceid=202, filesize=16.4 K
2024-11-16T16:38:30,012 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/43a91eb3dae84c8bb6b2ab71240c16a2 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/43a91eb3dae84c8bb6b2ab71240c16a2
2024-11-16T16:38:30,012 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=9.46 KB/9684 for 5ca511962c567b6c405af457006a4a3e in 26ms, sequenceid=202, compaction requested=false
2024-11-16T16:38:30,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:30,017 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 43a91eb3dae84c8bb6b2ab71240c16a2(size=88.6 K), total size for store is 105.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:30,018 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:30,018 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775109986; duration=0sec
2024-11-16T16:38:30,018 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:30,018 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:30,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:30,725 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:31,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:31,726 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:32,013 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-16T16:38:32,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7f033bf27acd46c08a847e25a1d6a7f7 is 1080, key is row0152/info:/1731775109987/Put/seqid=0
2024-11-16T16:38:32,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741865_1041 (size=15750)
2024-11-16T16:38:32,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741865_1041 (size=15750)
2024-11-16T16:38:32,026 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7f033bf27acd46c08a847e25a1d6a7f7
2024-11-16T16:38:32,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/7f033bf27acd46c08a847e25a1d6a7f7 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7
2024-11-16T16:38:32,037 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7, entries=10, sequenceid=216, filesize=15.4 K
2024-11-16T16:38:32,038 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=10.51 KB/10760 for 5ca511962c567b6c405af457006a4a3e in 26ms, sequenceid=216, compaction requested=true
2024-11-16T16:38:32,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:32,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:32,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:32,038 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:32,039 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123343 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:32,039 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:32,039 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:32,039 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/43a91eb3dae84c8bb6b2ab71240c16a2, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=120.5 K
2024-11-16T16:38:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:32,040 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:32,040 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a91eb3dae84c8bb6b2ab71240c16a2, keycount=79, bloomtype=ROW, size=88.6 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731775093648
2024-11-16T16:38:32,040 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting dada926439204103b0e6d821a381abdf, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731775109967
2024-11-16T16:38:32,040 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f033bf27acd46c08a847e25a1d6a7f7, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731775109987
2024-11-16T16:38:32,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/98c85e12ce8543278007be05322ac912 is 1080, key is row0162/info:/1731775112014/Put/seqid=0
2024-11-16T16:38:32,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741866_1042 (size=17906)
2024-11-16T16:38:32,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741866_1042 (size=17906)
2024-11-16T16:38:32,048 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/98c85e12ce8543278007be05322ac912
2024-11-16T16:38:32,052 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#83 average throughput is 51.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:32,052 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/4bf937f82d8647db94913a8a019e50d6 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:32,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/98c85e12ce8543278007be05322ac912 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912
2024-11-16T16:38:32,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741867_1043 (size=113509)
2024-11-16T16:38:32,056 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741867_1043 (size=113509)
2024-11-16T16:38:32,061 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912, entries=12, sequenceid=231, filesize=17.5 K
2024-11-16T16:38:32,062 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 5ca511962c567b6c405af457006a4a3e in 22ms, sequenceid=231, compaction requested=false
2024-11-16T16:38:32,062 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/4bf937f82d8647db94913a8a019e50d6 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/4bf937f82d8647db94913a8a019e50d6
2024-11-16T16:38:32,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:32,063 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:32,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/8ab5342c59814bd4a72720e94303392c is 1080, key is row0174/info:/1731775112041/Put/seqid=0
2024-11-16T16:38:32,068 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 4bf937f82d8647db94913a8a019e50d6(size=110.8 K), total size for store is 128.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:32,068 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:32,068 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775112038; duration=0sec
2024-11-16T16:38:32,068 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:32,068 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:32,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741868_1044 (size=17906)
2024-11-16T16:38:32,073 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741868_1044 (size=17906)
2024-11-16T16:38:32,476 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/8ab5342c59814bd4a72720e94303392c
2024-11-16T16:38:32,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/8ab5342c59814bd4a72720e94303392c as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c
2024-11-16T16:38:32,494 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c, entries=12, sequenceid=246, filesize=17.5 K
2024-11-16T16:38:32,495 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=7.36 KB/7532 for 5ca511962c567b6c405af457006a4a3e in 432ms, sequenceid=246, compaction requested=true
2024-11-16T16:38:32,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:32,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:32,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:32,495 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:32,497 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 149321 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:32,497 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:32,497 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:32,497 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/4bf937f82d8647db94913a8a019e50d6, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=145.8 K
2024-11-16T16:38:32,497 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bf937f82d8647db94913a8a019e50d6, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731775093648
2024-11-16T16:38:32,498 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c85e12ce8543278007be05322ac912, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731775112014
2024-11-16T16:38:32,498 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab5342c59814bd4a72720e94303392c, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731775112041
2024-11-16T16:38:32,510 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#85 average throughput is 42.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:32,511 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/278a12cf3fbb49bebcb795da6c71430b is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:32,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741869_1045 (size=139688)
2024-11-16T16:38:32,513 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741869_1045 (size=139688)
2024-11-16T16:38:32,519 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/278a12cf3fbb49bebcb795da6c71430b as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/278a12cf3fbb49bebcb795da6c71430b
2024-11-16T16:38:32,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:32,536 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 278a12cf3fbb49bebcb795da6c71430b(size=136.4 K), total size for store is 136.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:32,536 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:32,536 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775112495; duration=0sec
2024-11-16T16:38:32,536 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:32,536 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:32,727 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:33,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:33,728 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:34,085 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-16T16:38:34,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/84c6b5f10b4a4243b7f81d0e0fd94bca is 1080, key is row0186/info:/1731775112065/Put/seqid=0
2024-11-16T16:38:34,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741870_1046 (size=13595)
2024-11-16T16:38:34,097 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741870_1046 (size=13595)
2024-11-16T16:38:34,098 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/84c6b5f10b4a4243b7f81d0e0fd94bca
2024-11-16T16:38:34,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/84c6b5f10b4a4243b7f81d0e0fd94bca as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca
2024-11-16T16:38:34,109 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca, entries=8, sequenceid=259, filesize=13.3 K
2024-11-16T16:38:34,110 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=11.56 KB/11836 for 5ca511962c567b6c405af457006a4a3e in 26ms, sequenceid=259, compaction requested=false
2024-11-16T16:38:34,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:34,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:34,111 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:34,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/fa7b36ab561f451f863f08819b4e74da is 1080, key is row0194/info:/1731775114087/Put/seqid=0
2024-11-16T16:38:34,122 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741871_1047 (size=17918)
2024-11-16T16:38:34,123 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741871_1047 (size=17918)
2024-11-16T16:38:34,123 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/fa7b36ab561f451f863f08819b4e74da
2024-11-16T16:38:34,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/fa7b36ab561f451f863f08819b4e74da as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da
2024-11-16T16:38:34,133 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da, entries=12, sequenceid=274, filesize=17.5 K
2024-11-16T16:38:34,133 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=12.61 KB/12912 for 5ca511962c567b6c405af457006a4a3e in 22ms, sequenceid=274, compaction requested=true
2024-11-16T16:38:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:34,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:34,134 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:34,134 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-16T16:38:34,135 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 171201 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:34,135 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:34,135 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:34,135 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/278a12cf3fbb49bebcb795da6c71430b, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=167.2 K
2024-11-16T16:38:34,135 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 278a12cf3fbb49bebcb795da6c71430b, keycount=124, bloomtype=ROW, size=136.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731775093648
2024-11-16T16:38:34,136 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84c6b5f10b4a4243b7f81d0e0fd94bca, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731775112065
2024-11-16T16:38:34,136 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7b36ab561f451f863f08819b4e74da, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731775114087
2024-11-16T16:38:34,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/20d4b5f6bd3448b8bfbac270d96ab914 is 1080, key is row0206/info:/1731775114112/Put/seqid=0
2024-11-16T16:38:34,142 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741872_1048 (size=19013)
2024-11-16T16:38:34,143 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741872_1048 (size=19013)
2024-11-16T16:38:34,143 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/20d4b5f6bd3448b8bfbac270d96ab914
2024-11-16T16:38:34,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/20d4b5f6bd3448b8bfbac270d96ab914 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914
2024-11-16T16:38:34,149 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#89 average throughput is 36.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:34,150 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/13be02ea0370400fb64387dfb57a15c8 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:34,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741873_1049 (size=161367)
2024-11-16T16:38:34,152 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741873_1049 (size=161367)
2024-11-16T16:38:34,153 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914, entries=13, sequenceid=290, filesize=18.6 K
2024-11-16T16:38:34,154 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=6.30 KB/6456 for 5ca511962c567b6c405af457006a4a3e in 20ms, sequenceid=290, compaction requested=false
2024-11-16T16:38:34,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:34,157 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/13be02ea0370400fb64387dfb57a15c8 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/13be02ea0370400fb64387dfb57a15c8
2024-11-16T16:38:34,162 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 13be02ea0370400fb64387dfb57a15c8(size=157.6 K), total size for store is 176.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:34,162 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:34,162 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775114134; duration=0sec
2024-11-16T16:38:34,162 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:34,162 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:34,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:34,729 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:35,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:35,730 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:36,152 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-16T16:38:36,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/ecf78fad001842dbb00de16760ae2ead is 1080, key is row0219/info:/1731775114136/Put/seqid=0
2024-11-16T16:38:36,165 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741874_1050 (size=12523)
2024-11-16T16:38:36,166 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741874_1050 (size=12523)
2024-11-16T16:38:36,166 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/ecf78fad001842dbb00de16760ae2ead
2024-11-16T16:38:36,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/ecf78fad001842dbb00de16760ae2ead as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead
2024-11-16T16:38:36,177 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead, entries=7, sequenceid=301, filesize=12.2 K
2024-11-16T16:38:36,178 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=11.56 KB/11836 for 5ca511962c567b6c405af457006a4a3e in 26ms, sequenceid=301, compaction requested=true
2024-11-16T16:38:36,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:36,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:36,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:36,178 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:36,179 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 192903 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:36,180 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:36,180 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:36,180 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/13be02ea0370400fb64387dfb57a15c8, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=188.4 K
2024-11-16T16:38:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:36,180 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB
2024-11-16T16:38:36,180 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13be02ea0370400fb64387dfb57a15c8, keycount=144, bloomtype=ROW, size=157.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731775093648
2024-11-16T16:38:36,181 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d4b5f6bd3448b8bfbac270d96ab914, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731775114112
2024-11-16T16:38:36,181 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf78fad001842dbb00de16760ae2ead, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731775114136
2024-11-16T16:38:36,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/b9cd65248f414ad3a5ba792ee416a251 is 1080, key is row0226/info:/1731775116155/Put/seqid=0
2024-11-16T16:38:36,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741875_1051 (size=17918)
2024-11-16T16:38:36,188 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741875_1051 (size=17918)
2024-11-16T16:38:36,189 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/b9cd65248f414ad3a5ba792ee416a251
2024-11-16T16:38:36,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/b9cd65248f414ad3a5ba792ee416a251 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251
2024-11-16T16:38:36,196 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#92 average throughput is 56.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:36,197 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/9d8df4049d2d454a811e7a54add09ca9 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:36,199 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741876_1052 (size=183053)
2024-11-16T16:38:36,200 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741876_1052 (size=183053)
2024-11-16T16:38:36,200 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251, entries=12, sequenceid=316, filesize=17.5 K
2024-11-16T16:38:36,201 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=10.51 KB/10760 for 5ca511962c567b6c405af457006a4a3e in 21ms, sequenceid=316, compaction requested=false
2024-11-16T16:38:36,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36337 {}] regionserver.HRegion(8855): Flush requested on 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:36,201 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-16T16:38:36,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a3c6e04519ad4afab536d46878a237ab is 1080, key is row0238/info:/1731775116181/Put/seqid=0
2024-11-16T16:38:36,205 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/9d8df4049d2d454a811e7a54add09ca9 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/9d8df4049d2d454a811e7a54add09ca9
2024-11-16T16:38:36,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741877_1053 (size=16839)
2024-11-16T16:38:36,209 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741877_1053 (size=16839)
2024-11-16T16:38:36,209 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a3c6e04519ad4afab536d46878a237ab
2024-11-16T16:38:36,210 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 9d8df4049d2d454a811e7a54add09ca9(size=178.8 K), total size for store is 196.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:36,210 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:36,210 INFO  [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775116178; duration=0sec
2024-11-16T16:38:36,210 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:36,210 DEBUG [RS:0;303fe01965e8:36337-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a3c6e04519ad4afab536d46878a237ab as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab
2024-11-16T16:38:36,219 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab, entries=11, sequenceid=330, filesize=16.4 K
2024-11-16T16:38:36,220 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=8.41 KB/8608 for 5ca511962c567b6c405af457006a4a3e in 19ms, sequenceid=330, compaction requested=true
2024-11-16T16:38:36,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:36,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5ca511962c567b6c405af457006a4a3e:info, priority=-2147483648, current under compaction store size is 1
2024-11-16T16:38:36,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:36,220 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-16T16:38:36,221 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 217810 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-16T16:38:36,221 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1541): 5ca511962c567b6c405af457006a4a3e/info is initiating minor compaction (all files)
2024-11-16T16:38:36,221 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5ca511962c567b6c405af457006a4a3e/info in TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:36,221 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/9d8df4049d2d454a811e7a54add09ca9, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab] into tmpdir=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp, totalSize=212.7 K
2024-11-16T16:38:36,221 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8df4049d2d454a811e7a54add09ca9, keycount=164, bloomtype=ROW, size=178.8 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731775093648
2024-11-16T16:38:36,222 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting b9cd65248f414ad3a5ba792ee416a251, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731775116155
2024-11-16T16:38:36,222 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c6e04519ad4afab536d46878a237ab, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731775116181
2024-11-16T16:38:36,231 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5ca511962c567b6c405af457006a4a3e#info#compaction#94 average throughput is 63.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-16T16:38:36,232 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/25ae31da69204731aba5a203ddb711b5 is 1080, key is row0062/info:/1731775093648/Put/seqid=0
2024-11-16T16:38:36,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741878_1054 (size=208049)
2024-11-16T16:38:36,234 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741878_1054 (size=208049)
2024-11-16T16:38:36,240 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/25ae31da69204731aba5a203ddb711b5 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/25ae31da69204731aba5a203ddb711b5
2024-11-16T16:38:36,246 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5ca511962c567b6c405af457006a4a3e/info of 5ca511962c567b6c405af457006a4a3e into 25ae31da69204731aba5a203ddb711b5(size=203.2 K), total size for store is 203.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-16T16:38:36,246 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:36,246 INFO  [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., storeName=5ca511962c567b6c405af457006a4a3e/info, priority=13, startTime=1731775116220; duration=0sec
2024-11-16T16:38:36,246 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-16T16:38:36,246 DEBUG [RS:0;303fe01965e8:36337-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5ca511962c567b6c405af457006a4a3e:info
2024-11-16T16:38:36,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:36,731 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:37,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:37,732 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:38,217 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-16T16:38:38,218 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36337%2C1731775080237.1731775118217
2024-11-16T16:38:38,222 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,223 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,223 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,223 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,223 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,223 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775080867 with entries=318, filesize=310.38 KB; new WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775118217
2024-11-16T16:38:38,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741833_1009 (size=317837)
2024-11-16T16:38:38,225 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741833_1009 (size=317837)
2024-11-16T16:38:38,232 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34751:34751),(127.0.0.1/127.0.0.1:36855:36855)]
2024-11-16T16:38:38,235 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 5ca511962c567b6c405af457006a4a3e 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB
2024-11-16T16:38:38,241 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a7ebf2bc36754ee3a678043c4e9aeb5a is 1080, key is row0249/info:/1731775116203/Put/seqid=0
2024-11-16T16:38:38,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741880_1056 (size=13602)
2024-11-16T16:38:38,245 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741880_1056 (size=13602)
2024-11-16T16:38:38,246 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a7ebf2bc36754ee3a678043c4e9aeb5a
2024-11-16T16:38:38,251 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/.tmp/info/a7ebf2bc36754ee3a678043c4e9aeb5a as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a7ebf2bc36754ee3a678043c4e9aeb5a
2024-11-16T16:38:38,256 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a7ebf2bc36754ee3a678043c4e9aeb5a, entries=8, sequenceid=343, filesize=13.3 K
2024-11-16T16:38:38,257 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=0 B/0 for 5ca511962c567b6c405af457006a4a3e in 23ms, sequenceid=343, compaction requested=false
2024-11-16T16:38:38,257 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5ca511962c567b6c405af457006a4a3e:

2024-11-16T16:38:38,257 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-16T16:38:38,261 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/6df1e87ebcdf418192bfca64f3df63a6 is 186, key is TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d./info:regioninfo/1731775096556/Put/seqid=0
2024-11-16T16:38:38,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741881_1057 (size=6153)
2024-11-16T16:38:38,265 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741881_1057 (size=6153)
2024-11-16T16:38:38,266 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/6df1e87ebcdf418192bfca64f3df63a6
2024-11-16T16:38:38,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/.tmp/info/6df1e87ebcdf418192bfca64f3df63a6 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/info/6df1e87ebcdf418192bfca64f3df63a6
2024-11-16T16:38:38,275 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/info/6df1e87ebcdf418192bfca64f3df63a6, entries=5, sequenceid=21, filesize=6.0 K
2024-11-16T16:38:38,276 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 19ms, sequenceid=21, compaction requested=false
2024-11-16T16:38:38,276 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-16T16:38:38,276 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ccab8207ddcd64af7f8730b3b8374c0d:

2024-11-16T16:38:38,277 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C36337%2C1731775080237.1731775118277
2024-11-16T16:38:38,281 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,281 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,281 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,281 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,281 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,281 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775118217 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775118277
2024-11-16T16:38:38,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741879_1055 (size=731)
2024-11-16T16:38:38,283 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741879_1055 (size=731)
2024-11-16T16:38:38,284 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775080867 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs/303fe01965e8%2C36337%2C1731775080237.1731775080867
2024-11-16T16:38:38,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34751:34751),(127.0.0.1/127.0.0.1:36855:36855)]
2024-11-16T16:38:38,285 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-16T16:38:38,285 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/WALs/303fe01965e8,36337,1731775080237/303fe01965e8%2C36337%2C1731775080237.1731775118217 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs/303fe01965e8%2C36337%2C1731775080237.1731775118217
2024-11-16T16:38:38,286 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:38:38,286 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:38:38,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:38,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:38,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:38,286 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:38:38,286 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:38:38,286 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1071097855, stopped=false
2024-11-16T16:38:38,286 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,35141,1731775080035
2024-11-16T16:38:38,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:38,330 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:38,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:38,330 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:38,330 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:38:38,330 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:38:38,330 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:38,330 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:38,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:38,331 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,36337,1731775080237' *****
2024-11-16T16:38:38,331 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:38:38,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:38,331 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:38:38,331 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:38:38,331 INFO  [RS:0;303fe01965e8:36337 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:38:38,331 INFO  [RS:0;303fe01965e8:36337 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:38:38,331 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(3091): Received CLOSE for 5ca511962c567b6c405af457006a4a3e
2024-11-16T16:38:38,331 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(3091): Received CLOSE for ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,36337,1731775080237
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5ca511962c567b6c405af457006a4a3e, disabling compactions & flushes
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:36337.
2024-11-16T16:38:38,332 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:38,332 DEBUG [RS:0;303fe01965e8:36337 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:38,332 DEBUG [RS:0;303fe01965e8:36337 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e. after waiting 0 ms
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:38:38,332 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-16T16:38:38,332 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1325): Online Regions={5ca511962c567b6c405af457006a4a3e=TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e., 1588230740=hbase:meta,,1.1588230740, ccab8207ddcd64af7f8730b3b8374c0d=TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.}
2024-11-16T16:38:38,332 DEBUG [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5ca511962c567b6c405af457006a4a3e, ccab8207ddcd64af7f8730b3b8374c0d
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:38:38,332 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:38:38,332 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:38:38,332 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-top, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7468a952e0de4b3c93eb34a7ee834213, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a39d34d03bc645d1b0b6d6108c02b3c3, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/43a91eb3dae84c8bb6b2ab71240c16a2, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/4bf937f82d8647db94913a8a019e50d6, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/278a12cf3fbb49bebcb795da6c71430b, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/13be02ea0370400fb64387dfb57a15c8, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/9d8df4049d2d454a811e7a54add09ca9, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab] to archive
2024-11-16T16:38:38,334 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:38:38,335 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:38,337 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-16T16:38:38,337 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-cbee1cd6927f47b78a68a9ca9ce0fb20
2024-11-16T16:38:38,337 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:38:38,337 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:38,337 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775118332Running coprocessor pre-close hooks at 1731775118332Disabling compacts and flushes for region at 1731775118332Disabling writes for close at 1731775118332Writing region close event to WAL at 1731775118334 (+2 ms)Running coprocessor post-close hooks at 1731775118337 (+3 ms)Closed at 1731775118337
2024-11-16T16:38:38,337 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:38,338 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7468a952e0de4b3c93eb34a7ee834213 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7468a952e0de4b3c93eb34a7ee834213
2024-11-16T16:38:38,339 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/TestLogRolling-testLogRolling=7529d189d7c11f7fb002d1d2a22cbda8-e33a136ecaff44339f1abff751face67
2024-11-16T16:38:38,340 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/e5f700046e0b4bacaae18496dffeace4
2024-11-16T16:38:38,341 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a39d34d03bc645d1b0b6d6108c02b3c3 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a39d34d03bc645d1b0b6d6108c02b3c3
2024-11-16T16:38:38,342 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/72496ae50fb64175a2c2f13aeb7ccdd2
2024-11-16T16:38:38,343 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7c8e6671aa0c450fbbbb954792933f09
2024-11-16T16:38:38,344 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/43a91eb3dae84c8bb6b2ab71240c16a2 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/43a91eb3dae84c8bb6b2ab71240c16a2
2024-11-16T16:38:38,345 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/199ab26e663d4b549a9829e136eff1ee
2024-11-16T16:38:38,346 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/dada926439204103b0e6d821a381abdf
2024-11-16T16:38:38,347 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/4bf937f82d8647db94913a8a019e50d6 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/4bf937f82d8647db94913a8a019e50d6
2024-11-16T16:38:38,348 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/7f033bf27acd46c08a847e25a1d6a7f7
2024-11-16T16:38:38,349 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/98c85e12ce8543278007be05322ac912
2024-11-16T16:38:38,350 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/278a12cf3fbb49bebcb795da6c71430b to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/278a12cf3fbb49bebcb795da6c71430b
2024-11-16T16:38:38,351 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/8ab5342c59814bd4a72720e94303392c
2024-11-16T16:38:38,352 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/84c6b5f10b4a4243b7f81d0e0fd94bca
2024-11-16T16:38:38,353 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/13be02ea0370400fb64387dfb57a15c8 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/13be02ea0370400fb64387dfb57a15c8
2024-11-16T16:38:38,354 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/fa7b36ab561f451f863f08819b4e74da
2024-11-16T16:38:38,355 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/20d4b5f6bd3448b8bfbac270d96ab914
2024-11-16T16:38:38,356 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/9d8df4049d2d454a811e7a54add09ca9 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/9d8df4049d2d454a811e7a54add09ca9
2024-11-16T16:38:38,357 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/ecf78fad001842dbb00de16760ae2ead
2024-11-16T16:38:38,358 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/b9cd65248f414ad3a5ba792ee416a251
2024-11-16T16:38:38,359 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/info/a3c6e04519ad4afab536d46878a237ab
2024-11-16T16:38:38,359 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=303fe01965e8:35141 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-16T16:38:38,359 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [7468a952e0de4b3c93eb34a7ee834213=42984, e5f700046e0b4bacaae18496dffeace4=12516, a39d34d03bc645d1b0b6d6108c02b3c3=63636, 72496ae50fb64175a2c2f13aeb7ccdd2=17906, 7c8e6671aa0c450fbbbb954792933f09=20078, 43a91eb3dae84c8bb6b2ab71240c16a2=90765, 199ab26e663d4b549a9829e136eff1ee=16828, dada926439204103b0e6d821a381abdf=16828, 4bf937f82d8647db94913a8a019e50d6=113509, 7f033bf27acd46c08a847e25a1d6a7f7=15750, 98c85e12ce8543278007be05322ac912=17906, 278a12cf3fbb49bebcb795da6c71430b=139688, 8ab5342c59814bd4a72720e94303392c=17906, 84c6b5f10b4a4243b7f81d0e0fd94bca=13595, 13be02ea0370400fb64387dfb57a15c8=161367, fa7b36ab561f451f863f08819b4e74da=17918, 20d4b5f6bd3448b8bfbac270d96ab914=19013, 9d8df4049d2d454a811e7a54add09ca9=183053, ecf78fad001842dbb00de16760ae2ead=12523, b9cd65248f414ad3a5ba792ee416a251=17918, a3c6e04519ad4afab536d46878a237ab=16839]
2024-11-16T16:38:38,363 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/5ca511962c567b6c405af457006a4a3e/recovered.edits/346.seqid, newMaxSeqId=346, maxSeqId=130
2024-11-16T16:38:38,363 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:38,363 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5ca511962c567b6c405af457006a4a3e:
Waiting for close lock at 1731775118332Running coprocessor pre-close hooks at 1731775118332Disabling compacts and flushes for region at 1731775118332Disabling writes for close at 1731775118332Writing region close event to WAL at 1731775118360 (+28 ms)Running coprocessor post-close hooks at 1731775118363 (+3 ms)Closed at 1731775118363
2024-11-16T16:38:38,363 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731775095762.5ca511962c567b6c405af457006a4a3e.
2024-11-16T16:38:38,363 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ccab8207ddcd64af7f8730b3b8374c0d, disabling compactions & flushes
2024-11-16T16:38:38,363 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:38,363 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:38,364 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d. after waiting 0 ms
2024-11-16T16:38:38,364 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:38,364 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8->hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/7529d189d7c11f7fb002d1d2a22cbda8/info/3bbcf780bb1347c29591a44dfd7f4bb0-bottom] to archive
2024-11-16T16:38:38,365 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-16T16:38:38,366 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8 to hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/archive/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/info/3bbcf780bb1347c29591a44dfd7f4bb0.7529d189d7c11f7fb002d1d2a22cbda8
2024-11-16T16:38:38,366 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-16T16:38:38,370 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/data/default/TestLogRolling-testLogRolling/ccab8207ddcd64af7f8730b3b8374c0d/recovered.edits/135.seqid, newMaxSeqId=135, maxSeqId=130
2024-11-16T16:38:38,370 INFO  [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:38,370 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ccab8207ddcd64af7f8730b3b8374c0d:
Waiting for close lock at 1731775118363Running coprocessor pre-close hooks at 1731775118363Disabling compacts and flushes for region at 1731775118363Disabling writes for close at 1731775118364 (+1 ms)Writing region close event to WAL at 1731775118366 (+2 ms)Running coprocessor post-close hooks at 1731775118370 (+4 ms)Closed at 1731775118370
2024-11-16T16:38:38,370 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731775095762.ccab8207ddcd64af7f8730b3b8374c0d.
2024-11-16T16:38:38,532 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,36337,1731775080237; all regions closed.
2024-11-16T16:38:38,534 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,534 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,534 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,534 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,534 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741834_1010 (size=8107)
2024-11-16T16:38:38,539 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741834_1010 (size=8107)
2024-11-16T16:38:38,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:38,545 DEBUG [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs
2024-11-16T16:38:38,545 INFO  [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C36337%2C1731775080237.meta:.meta(num 1731775081298)
2024-11-16T16:38:38,545 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,546 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,546 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,546 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,546 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,548 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741882_1058 (size=780)
2024-11-16T16:38:38,549 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741882_1058 (size=780)
2024-11-16T16:38:38,551 DEBUG [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/oldWALs
2024-11-16T16:38:38,551 INFO  [RS:0;303fe01965e8:36337 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C36337%2C1731775080237:(num 1731775118277)
2024-11-16T16:38:38,551 DEBUG [RS:0;303fe01965e8:36337 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:38,551 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:38:38,552 INFO  [RS:0;303fe01965e8:36337 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:38:38,552 INFO  [RS:0;303fe01965e8:36337 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-16T16:38:38,552 INFO  [RS:0;303fe01965e8:36337 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:38:38,552 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:38:38,552 INFO  [RS:0;303fe01965e8:36337 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36337
2024-11-16T16:38:38,561 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,36337,1731775080237
2024-11-16T16:38:38,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:38:38,561 INFO  [RS:0;303fe01965e8:36337 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:38:38,572 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,36337,1731775080237]
2024-11-16T16:38:38,582 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,36337,1731775080237 already deleted, retry=false
2024-11-16T16:38:38,582 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,36337,1731775080237 expired; onlineServers=0
2024-11-16T16:38:38,583 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,35141,1731775080035' *****
2024-11-16T16:38:38,583 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:38:38,583 INFO  [M:0;303fe01965e8:35141 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:38:38,583 INFO  [M:0;303fe01965e8:35141 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:38:38,583 DEBUG [M:0;303fe01965e8:35141 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:38:38,583 DEBUG [M:0;303fe01965e8:35141 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:38:38,584 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:38:38,584 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775080632 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775080632,5,FailOnTimeoutGroup]
2024-11-16T16:38:38,584 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775080632 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775080632,5,FailOnTimeoutGroup]
2024-11-16T16:38:38,584 INFO  [M:0;303fe01965e8:35141 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:38:38,584 INFO  [M:0;303fe01965e8:35141 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:38:38,584 DEBUG [M:0;303fe01965e8:35141 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:38:38,584 INFO  [M:0;303fe01965e8:35141 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:38:38,585 INFO  [M:0;303fe01965e8:35141 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:38:38,585 INFO  [M:0;303fe01965e8:35141 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:38:38,585 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:38:38,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:38:38,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:38,593 DEBUG [M:0;303fe01965e8:35141 {}] zookeeper.ZKUtil(347): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:38:38,594 WARN  [M:0;303fe01965e8:35141 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:38:38,595 INFO  [M:0;303fe01965e8:35141 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/.lastflushedseqids
2024-11-16T16:38:38,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741883_1059 (size=228)
2024-11-16T16:38:38,600 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741883_1059 (size=228)
2024-11-16T16:38:38,600 INFO  [M:0;303fe01965e8:35141 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:38:38,600 INFO  [M:0;303fe01965e8:35141 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:38:38,600 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:38:38,600 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:38,600 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:38,600 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:38:38,601 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:38,601 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.38 KB
2024-11-16T16:38:38,617 DEBUG [M:0;303fe01965e8:35141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a5cf972181fd4fad9e63c32f09d9f039 is 82, key is hbase:meta,,1/info:regioninfo/1731775081330/Put/seqid=0
2024-11-16T16:38:38,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741884_1060 (size=5672)
2024-11-16T16:38:38,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741884_1060 (size=5672)
2024-11-16T16:38:38,621 INFO  [M:0;303fe01965e8:35141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a5cf972181fd4fad9e63c32f09d9f039
2024-11-16T16:38:38,639 DEBUG [M:0;303fe01965e8:35141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d8bb43d4eeee4b1c8608ab5d54833174 is 750, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731775081866/Put/seqid=0
2024-11-16T16:38:38,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741885_1061 (size=7090)
2024-11-16T16:38:38,644 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741885_1061 (size=7090)
2024-11-16T16:38:38,644 INFO  [M:0;303fe01965e8:35141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d8bb43d4eeee4b1c8608ab5d54833174
2024-11-16T16:38:38,648 INFO  [M:0;303fe01965e8:35141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d8bb43d4eeee4b1c8608ab5d54833174
2024-11-16T16:38:38,663 DEBUG [M:0;303fe01965e8:35141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83a86854aee9422c93725ea9fffaa82e is 69, key is 303fe01965e8,36337,1731775080237/rs:state/1731775080711/Put/seqid=0
2024-11-16T16:38:38,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741886_1062 (size=5156)
2024-11-16T16:38:38,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741886_1062 (size=5156)
2024-11-16T16:38:38,667 INFO  [M:0;303fe01965e8:35141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83a86854aee9422c93725ea9fffaa82e
2024-11-16T16:38:38,672 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:38,672 INFO  [RS:0;303fe01965e8:36337 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:38:38,672 DEBUG [pool-823-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36337-0x1014482a1fd0001, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:38,672 INFO  [RS:0;303fe01965e8:36337 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,36337,1731775080237; zookeeper connection closed.
2024-11-16T16:38:38,672 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f67d0b5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f67d0b5
2024-11-16T16:38:38,673 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:38:38,683 DEBUG [M:0;303fe01965e8:35141 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/001806c10ed145298dac53c39e0b33d5 is 52, key is load_balancer_on/state:d/1731775081481/Put/seqid=0
2024-11-16T16:38:38,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741887_1063 (size=5056)
2024-11-16T16:38:38,688 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741887_1063 (size=5056)
2024-11-16T16:38:38,688 INFO  [M:0;303fe01965e8:35141 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/001806c10ed145298dac53c39e0b33d5
2024-11-16T16:38:38,692 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a5cf972181fd4fad9e63c32f09d9f039 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a5cf972181fd4fad9e63c32f09d9f039
2024-11-16T16:38:38,696 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a5cf972181fd4fad9e63c32f09d9f039, entries=8, sequenceid=125, filesize=5.5 K
2024-11-16T16:38:38,697 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d8bb43d4eeee4b1c8608ab5d54833174 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d8bb43d4eeee4b1c8608ab5d54833174
2024-11-16T16:38:38,700 INFO  [M:0;303fe01965e8:35141 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d8bb43d4eeee4b1c8608ab5d54833174
2024-11-16T16:38:38,700 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d8bb43d4eeee4b1c8608ab5d54833174, entries=13, sequenceid=125, filesize=6.9 K
2024-11-16T16:38:38,701 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/83a86854aee9422c93725ea9fffaa82e as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/83a86854aee9422c93725ea9fffaa82e
2024-11-16T16:38:38,704 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/83a86854aee9422c93725ea9fffaa82e, entries=1, sequenceid=125, filesize=5.0 K
2024-11-16T16:38:38,705 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/001806c10ed145298dac53c39e0b33d5 as hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/001806c10ed145298dac53c39e0b33d5
2024-11-16T16:38:38,709 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34573/user/jenkins/test-data/64a1b515-bd2a-5204-a002-617494d364ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/001806c10ed145298dac53c39e0b33d5, entries=1, sequenceid=125, filesize=4.9 K
2024-11-16T16:38:38,710 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=125, compaction requested=false
2024-11-16T16:38:38,711 INFO  [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:38,711 DEBUG [M:0;303fe01965e8:35141 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775118600Disabling compacts and flushes for region at 1731775118600Disabling writes for close at 1731775118600Obtaining lock to block concurrent updates at 1731775118601 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731775118601Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64840, getOffHeapSize=0, getCellsCount=148 at 1731775118601Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731775118601Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731775118602 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731775118616 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731775118617 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731775118625 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731775118639 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731775118639Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731775118648 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731775118662 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731775118662Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731775118671 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731775118683 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731775118683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72e3fbd8: reopening flushed file at 1731775118691 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11049a42: reopening flushed file at 1731775118696 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e9524cf: reopening flushed file at 1731775118700 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7afe9b64: reopening flushed file at 1731775118705 (+5 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.32 KB/64840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=125, compaction requested=false at 1731775118710 (+5 ms)Writing region close event to WAL at 1731775118711 (+1 ms)Closed at 1731775118711
2024-11-16T16:38:38,711 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,711 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,711 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,711 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,712 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:38,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43367 is added to blk_1073741830_1006 (size=61332)
2024-11-16T16:38:38,713 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45481 is added to blk_1073741830_1006 (size=61332)
2024-11-16T16:38:38,714 INFO  [M:0;303fe01965e8:35141 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:38:38,714 INFO  [M:0;303fe01965e8:35141 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35141
2024-11-16T16:38:38,714 INFO  [M:0;303fe01965e8:35141 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:38:38,714 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:38:38,732 INFO  [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:38:38,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:38,824 INFO  [M:0;303fe01965e8:35141 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:38:38,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:38,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35141-0x1014482a1fd0000, quorum=127.0.0.1:65180, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:38,853 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea981d3{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:38,854 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71c2dc3e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:38,854 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:38,855 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75eba18f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:38,855 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5169cded{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:38,858 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:38:38,858 WARN  [BP-192065853-172.17.0.2-1731775077407 heartbeating to localhost/127.0.0.1:34573 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:38:38,858 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:38:38,858 WARN  [BP-192065853-172.17.0.2-1731775077407 heartbeating to localhost/127.0.0.1:34573 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-192065853-172.17.0.2-1731775077407 (Datanode Uuid 9bbdc873-5fd2-4959-bed6-9f7b0296d9ab) service to localhost/127.0.0.1:34573
2024-11-16T16:38:38,859 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data3/current/BP-192065853-172.17.0.2-1731775077407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:38,859 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data4/current/BP-192065853-172.17.0.2-1731775077407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:38,859 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:38:38,861 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2079ccf2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:38,861 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13da1053{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:38,861 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:38,862 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13fa93c4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:38,862 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43cde2d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:38,863 WARN  [BP-192065853-172.17.0.2-1731775077407 heartbeating to localhost/127.0.0.1:34573 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:38:38,863 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:38:38,863 WARN  [BP-192065853-172.17.0.2-1731775077407 heartbeating to localhost/127.0.0.1:34573 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-192065853-172.17.0.2-1731775077407 (Datanode Uuid a80ad598-d00f-4457-bc9a-92d3bf96e6b7) service to localhost/127.0.0.1:34573
2024-11-16T16:38:38,863 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:38:38,864 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data1/current/BP-192065853-172.17.0.2-1731775077407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:38,864 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/cluster_a6f73b7a-a54e-fcbe-2d52-85f7d351d4d0/data/data2/current/BP-192065853-172.17.0.2-1731775077407 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:38,864 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:38:38,869 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5350cad{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:38:38,870 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@278f505{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:38,870 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:38,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ec6f4ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:38,870 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6150e164{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:38,876 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:38:38,904 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:38:38,913 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=231 (was 209)
Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:34573 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34573
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:34573
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:34573 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:34573 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=512 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=201 (was 176) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4275 (was 3462) - AvailableMemoryMB LEAK? -
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=231, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=201, ProcessCount=11, AvailableMemoryMB=4275
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.log.dir so I do NOT create it in target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8f3e8ad9-bada-04a8-fbf8-2cd6d4648e05/hadoop.tmp.dir so I do NOT create it in target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3, deleteOnExit=true
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-16T16:38:38,921 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/test.cache.data in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.tmp.dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-16T16:38:38,922 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-16T16:38:38,922 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/nfs.dump.dir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/java.io.tmpdir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-16T16:38:38,923 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-16T16:38:38,936 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:38:39,303 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:38:39,306 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:38:39,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:38:39,308 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:38:39,308 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:38:39,312 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:38:39,313 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7546f00d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:38:39,313 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6abd1419{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:38:39,407 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5c4027cc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/java.io.tmpdir/jetty-localhost-44649-hadoop-hdfs-3_4_1-tests_jar-_-any-7679099145382989305/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:38:39,407 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@51c3fa60{HTTP/1.1, (http/1.1)}{localhost:44649}
2024-11-16T16:38:39,407 INFO  [Time-limited test {}] server.Server(415): Started @295183ms
2024-11-16T16:38:39,418 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-16T16:38:39,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:39,661 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:38:39,664 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:38:39,664 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:38:39,665 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:38:39,665 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-16T16:38:39,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6adc48be{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:38:39,665 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a9c0f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:38:39,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:39,759 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a3cdc90{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/java.io.tmpdir/jetty-localhost-43549-hadoop-hdfs-3_4_1-tests_jar-_-any-14605947126599034737/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:39,760 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2a377aed{HTTP/1.1, (http/1.1)}{localhost:43549}
2024-11-16T16:38:39,760 INFO  [Time-limited test {}] server.Server(415): Started @295535ms
2024-11-16T16:38:39,761 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:38:39,785 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-16T16:38:39,787 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-16T16:38:39,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-16T16:38:39,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-16T16:38:39,788 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-16T16:38:39,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d05b3cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,AVAILABLE}
2024-11-16T16:38:39,790 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33455caf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-16T16:38:39,883 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e74a3b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/java.io.tmpdir/jetty-localhost-41193-hadoop-hdfs-3_4_1-tests_jar-_-any-8611809899242493152/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:39,883 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b1af14f{HTTP/1.1, (http/1.1)}{localhost:41193}
2024-11-16T16:38:39,883 INFO  [Time-limited test {}] server.Server(415): Started @295659ms
2024-11-16T16:38:39,884 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-16T16:38:40,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:40,733 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:41,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:38:41,189 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-16T16:38:41,190 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-16T16:38:41,191 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-16T16:38:41,232 WARN  [Thread-2521 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data1/current/BP-967819845-172.17.0.2-1731775118939/current, will proceed with Du for space computation calculation, 
2024-11-16T16:38:41,232 WARN  [Thread-2522 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data2/current/BP-967819845-172.17.0.2-1731775118939/current, will proceed with Du for space computation calculation, 
2024-11-16T16:38:41,249 WARN  [Thread-2485 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:38:41,251 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x91049d79439b6fc4 with lease ID 0x8afe852554ad3377: Processing first storage report for DS-081b6e89-dbd4-4f7b-abb4-7bd04aecce2e from datanode DatanodeRegistration(127.0.0.1:44397, datanodeUuid=798945b2-1c8a-47e8-84ae-8eb51c64a38a, infoPort=35441, infoSecurePort=0, ipcPort=34021, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939)
2024-11-16T16:38:41,251 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x91049d79439b6fc4 with lease ID 0x8afe852554ad3377: from storage DS-081b6e89-dbd4-4f7b-abb4-7bd04aecce2e node DatanodeRegistration(127.0.0.1:44397, datanodeUuid=798945b2-1c8a-47e8-84ae-8eb51c64a38a, infoPort=35441, infoSecurePort=0, ipcPort=34021, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:38:41,251 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x91049d79439b6fc4 with lease ID 0x8afe852554ad3377: Processing first storage report for DS-755ffe06-ebdf-41a2-9dd1-88c4aaeda231 from datanode DatanodeRegistration(127.0.0.1:44397, datanodeUuid=798945b2-1c8a-47e8-84ae-8eb51c64a38a, infoPort=35441, infoSecurePort=0, ipcPort=34021, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939)
2024-11-16T16:38:41,251 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x91049d79439b6fc4 with lease ID 0x8afe852554ad3377: from storage DS-755ffe06-ebdf-41a2-9dd1-88c4aaeda231 node DatanodeRegistration(127.0.0.1:44397, datanodeUuid=798945b2-1c8a-47e8-84ae-8eb51c64a38a, infoPort=35441, infoSecurePort=0, ipcPort=34021, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:38:41,379 WARN  [Thread-2532 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data3/current/BP-967819845-172.17.0.2-1731775118939/current, will proceed with Du for space computation calculation, 
2024-11-16T16:38:41,379 WARN  [Thread-2533 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data4/current/BP-967819845-172.17.0.2-1731775118939/current, will proceed with Du for space computation calculation, 
2024-11-16T16:38:41,397 WARN  [Thread-2508 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-16T16:38:41,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe578c4d6214222f9 with lease ID 0x8afe852554ad3378: Processing first storage report for DS-456a937e-9436-4eef-ab83-c32a303294b5 from datanode DatanodeRegistration(127.0.0.1:37663, datanodeUuid=5b49ba16-c286-4b60-8b34-043c0192a351, infoPort=45805, infoSecurePort=0, ipcPort=46179, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939)
2024-11-16T16:38:41,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe578c4d6214222f9 with lease ID 0x8afe852554ad3378: from storage DS-456a937e-9436-4eef-ab83-c32a303294b5 node DatanodeRegistration(127.0.0.1:37663, datanodeUuid=5b49ba16-c286-4b60-8b34-043c0192a351, infoPort=45805, infoSecurePort=0, ipcPort=46179, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:38:41,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe578c4d6214222f9 with lease ID 0x8afe852554ad3378: Processing first storage report for DS-4360092d-967c-4e42-bfab-f697d9aef868 from datanode DatanodeRegistration(127.0.0.1:37663, datanodeUuid=5b49ba16-c286-4b60-8b34-043c0192a351, infoPort=45805, infoSecurePort=0, ipcPort=46179, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939)
2024-11-16T16:38:41,399 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe578c4d6214222f9 with lease ID 0x8afe852554ad3378: from storage DS-4360092d-967c-4e42-bfab-f697d9aef868 node DatanodeRegistration(127.0.0.1:37663, datanodeUuid=5b49ba16-c286-4b60-8b34-043c0192a351, infoPort=45805, infoSecurePort=0, ipcPort=46179, storageInfo=lv=-57;cid=testClusterID;nsid=170149941;c=1731775118939), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-16T16:38:41,424 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c
2024-11-16T16:38:41,448 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/zookeeper_0, clientPort=63592, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-16T16:38:41,449 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63592
2024-11-16T16:38:41,449 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,452 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:38:41,465 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741825_1001 (size=7)
2024-11-16T16:38:41,467 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c with version=8
2024-11-16T16:38:41,467 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:36983/user/jenkins/test-data/df0d0cd1-db09-6aa0-13d5-b80cda0bf71c/hbase-staging
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:38:41,470 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-16T16:38:41,471 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:38:41,471 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39085
2024-11-16T16:38:41,473 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39085 connecting to ZooKeeper ensemble=127.0.0.1:63592
2024-11-16T16:38:41,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:41,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:390850x0, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:38:41,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39085-0x101448343d20000 connected
2024-11-16T16:38:41,633 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,635 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:41,639 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c, hbase.cluster.distributed=false
2024-11-16T16:38:41,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:38:41,643 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39085
2024-11-16T16:38:41,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39085
2024-11-16T16:38:41,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39085
2024-11-16T16:38:41,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39085
2024-11-16T16:38:41,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39085
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-16T16:38:41,662 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-16T16:38:41,663 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45279
2024-11-16T16:38:41,664 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45279 connecting to ZooKeeper ensemble=127.0.0.1:63592
2024-11-16T16:38:41,664 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,665 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452790x0, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-16T16:38:41,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452790x0, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:41,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45279-0x101448343d20001 connected
2024-11-16T16:38:41,676 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-16T16:38:41,677 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-16T16:38:41,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-16T16:38:41,678 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-16T16:38:41,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45279
2024-11-16T16:38:41,682 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45279
2024-11-16T16:38:41,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45279
2024-11-16T16:38:41,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45279
2024-11-16T16:38:41,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45279
2024-11-16T16:38:41,696 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:39085
2024-11-16T16:38:41,696 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:41,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:41,707 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-16T16:38:41,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,718 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-16T16:38:41,718 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,39085,1731775121470 from backup master directory
2024-11-16T16:38:41,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:41,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-16T16:38:41,728 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:38:41,728 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,731 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/hbase.id] with ID: bdd5b43e-59c5-4f24-ae8a-d95771b689f4
2024-11-16T16:38:41,731 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/.tmp/hbase.id
2024-11-16T16:38:41,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:41,736 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:38:41,737 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741826_1002 (size=42)
2024-11-16T16:38:41,738 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/.tmp/hbase.id]:[hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/hbase.id]
2024-11-16T16:38:41,749 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:41,749 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-16T16:38:41,750 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-16T16:38:41,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:38:41,765 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741827_1003 (size=196)
2024-11-16T16:38:41,765 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-16T16:38:41,766 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-16T16:38:41,766 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:41,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:38:41,772 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741828_1004 (size=1189)
2024-11-16T16:38:41,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store
2024-11-16T16:38:41,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:38:41,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741829_1005 (size=34)
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:38:41,779 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:41,779 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:41,779 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775121779Disabling compacts and flushes for region at 1731775121779Disabling writes for close at 1731775121779Writing region close event to WAL at 1731775121779Closed at 1731775121779
2024-11-16T16:38:41,780 WARN  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/.initializing
2024-11-16T16:38:41,780 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/WALs/303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,782 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C39085%2C1731775121470, suffix=, logDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/WALs/303fe01965e8,39085,1731775121470, archiveDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/oldWALs, maxLogs=10
2024-11-16T16:38:41,782 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C39085%2C1731775121470.1731775121782
2024-11-16T16:38:41,786 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/WALs/303fe01965e8,39085,1731775121470/303fe01965e8%2C39085%2C1731775121470.1731775121782
2024-11-16T16:38:41,786 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35441:35441),(127.0.0.1/127.0.0.1:45805:45805)]
2024-11-16T16:38:41,787 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:38:41,787 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:41,787 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,787 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,788 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:41,789 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-16T16:38:41,789 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:41,789 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:41,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:41,790 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-16T16:38:41,790 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:41,791 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:41,791 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:41,791 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-16T16:38:41,792 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:41,792 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:41,792 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-16T16:38:41,793 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-16T16:38:41,793 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:41,793 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-16T16:38:41,793 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,794 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,794 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,795 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-16T16:38:41,796 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-16T16:38:41,798 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:38:41,798 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868357, jitterRate=0.10417348146438599}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-16T16:38:41,798 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731775121787Initializing all the Stores at 1731775121788 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775121788Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775121788Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775121788Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775121788Cleaning up temporary data from old regions at 1731775121795 (+7 ms)Region opened successfully at 1731775121798 (+3 ms)
2024-11-16T16:38:41,799 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-16T16:38:41,801 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405b1ee7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:38:41,802 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-16T16:38:41,802 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-16T16:38:41,802 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-16T16:38:41,802 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-16T16:38:41,803 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-16T16:38:41,803 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-16T16:38:41,803 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-16T16:38:41,805 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-16T16:38:41,806 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-16T16:38:41,812 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-16T16:38:41,812 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-16T16:38:41,813 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-16T16:38:41,822 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-16T16:38:41,823 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-16T16:38:41,824 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-16T16:38:41,833 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-16T16:38:41,834 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-16T16:38:41,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-16T16:38:41,848 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-16T16:38:41,854 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-16T16:38:41,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:41,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:41,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,865 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,39085,1731775121470, sessionid=0x101448343d20000, setting cluster-up flag (Was=false)
2024-11-16T16:38:41,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,917 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-16T16:38:41,919 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:41,970 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-16T16:38:41,974 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,39085,1731775121470
2024-11-16T16:38:41,978 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-16T16:38:41,980 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:41,980 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-16T16:38:41,980 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-16T16:38:41,981 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,39085,1731775121470
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-16T16:38:41,982 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:41,982 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:41,982 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:41,982 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5
2024-11-16T16:38:41,983 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10
2024-11-16T16:38:41,983 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:41,983 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:38:41,983 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:41,983 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731775151983
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-16T16:38:41,984 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:41,984 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-16T16:38:41,984 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-16T16:38:41,985 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-16T16:38:41,985 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-16T16:38:41,985 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775121985,5,FailOnTimeoutGroup]
2024-11-16T16:38:41,985 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:41,985 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-16T16:38:41,986 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775121985,5,FailOnTimeoutGroup]
2024-11-16T16:38:41,986 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:41,986 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-16T16:38:41,986 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:41,986 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:41,988 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(746): ClusterId : bdd5b43e-59c5-4f24-ae8a-d95771b689f4
2024-11-16T16:38:41,988 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-16T16:38:41,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:38:41,991 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741831_1007 (size=1321)
2024-11-16T16:38:41,992 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-16T16:38:41,992 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c
2024-11-16T16:38:41,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:38:41,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741832_1008 (size=32)
2024-11-16T16:38:42,000 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:42,000 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-16T16:38:42,000 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-16T16:38:42,001 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:38:42,002 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:38:42,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,002 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,002 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:38:42,003 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:38:42,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,004 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,004 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:38:42,005 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:38:42,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,005 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,005 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:38:42,006 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:38:42,006 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,007 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:38:42,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740
2024-11-16T16:38:42,008 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740
2024-11-16T16:38:42,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:38:42,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:38:42,011 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:38:42,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:38:42,013 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-16T16:38:42,013 DEBUG [RS:0;303fe01965e8:45279 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34175d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0
2024-11-16T16:38:42,015 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-16T16:38:42,015 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754432, jitterRate=-0.040690675377845764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:38:42,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731775122000Initializing all the Stores at 1731775122000Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122000Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122001 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775122001Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122001Cleaning up temporary data from old regions at 1731775122010 (+9 ms)Region opened successfully at 1731775122016 (+6 ms)
2024-11-16T16:38:42,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:38:42,016 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:38:42,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:38:42,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:38:42,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:38:42,017 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:42,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775122016Disabling compacts and flushes for region at 1731775122016Disabling writes for close at 1731775122016Writing region close event to WAL at 1731775122017 (+1 ms)Closed at 1731775122017
2024-11-16T16:38:42,018 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:42,018 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-16T16:38:42,018 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-16T16:38:42,020 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:38:42,021 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-16T16:38:42,029 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:45279
2024-11-16T16:38:42,029 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-16T16:38:42,029 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-16T16:38:42,029 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-16T16:38:42,030 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,39085,1731775121470 with port=45279, startcode=1731775121662
2024-11-16T16:38:42,030 DEBUG [RS:0;303fe01965e8:45279 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-16T16:38:42,032 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50913, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-16T16:38:42,032 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39085 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,032 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39085 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,033 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c
2024-11-16T16:38:42,033 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44701
2024-11-16T16:38:42,033 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-16T16:38:42,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:38:42,044 DEBUG [RS:0;303fe01965e8:45279 {}] zookeeper.ZKUtil(111): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,044 WARN  [RS:0;303fe01965e8:45279 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-16T16:38:42,044 INFO  [RS:0;303fe01965e8:45279 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:42,044 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,044 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,45279,1731775121662]
2024-11-16T16:38:42,047 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-16T16:38:42,048 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-16T16:38:42,049 INFO  [RS:0;303fe01965e8:45279 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-16T16:38:42,049 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,049 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:38:42,050 DEBUG [RS:0;303fe01965e8:45279 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,050 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,051 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,051 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45279,1731775121662-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:38:42,064 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-16T16:38:42,064 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45279,1731775121662-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,064 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,064 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.Replication(171): 303fe01965e8,45279,1731775121662 started
2024-11-16T16:38:42,076 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,076 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,45279,1731775121662, RpcServer on 303fe01965e8/172.17.0.2:45279, sessionid=0x101448343d20001
2024-11-16T16:38:42,076 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-16T16:38:42,076 DEBUG [RS:0;303fe01965e8:45279 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,076 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,45279,1731775121662'
2024-11-16T16:38:42,076 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,45279,1731775121662'
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-16T16:38:42,077 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-16T16:38:42,078 DEBUG [RS:0;303fe01965e8:45279 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-16T16:38:42,078 INFO  [RS:0;303fe01965e8:45279 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-16T16:38:42,078 INFO  [RS:0;303fe01965e8:45279 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-16T16:38:42,171 WARN  [303fe01965e8:39085 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-16T16:38:42,181 INFO  [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C45279%2C1731775121662, suffix=, logDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/303fe01965e8,45279,1731775121662, archiveDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs, maxLogs=32
2024-11-16T16:38:42,182 INFO  [RS:0;303fe01965e8:45279 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C45279%2C1731775121662.1731775122181
2024-11-16T16:38:42,190 INFO  [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/303fe01965e8,45279,1731775121662/303fe01965e8%2C45279%2C1731775121662.1731775122181
2024-11-16T16:38:42,192 DEBUG [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35441:35441),(127.0.0.1/127.0.0.1:45805:45805)]
2024-11-16T16:38:42,422 DEBUG [303fe01965e8:39085 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-16T16:38:42,422 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,425 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,45279,1731775121662, state=OPENING
2024-11-16T16:38:42,475 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-16T16:38:42,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:42,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:42,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:42,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:42,488 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-16T16:38:42,488 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,45279,1731775121662}]
2024-11-16T16:38:42,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.meta.1731774935631.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:42,644 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-16T16:38:42,647 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60439, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-16T16:38:42,652 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-16T16:38:42,652 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:42,654 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C45279%2C1731775121662.meta, suffix=.meta, logDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/303fe01965e8,45279,1731775121662, archiveDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs, maxLogs=32
2024-11-16T16:38:42,655 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor 303fe01965e8%2C45279%2C1731775121662.meta.1731775122655.meta
2024-11-16T16:38:42,661 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/303fe01965e8,45279,1731775121662/303fe01965e8%2C45279%2C1731775121662.meta.1731775122655.meta
2024-11-16T16:38:42,664 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45805:45805),(127.0.0.1/127.0.0.1:35441:35441)]
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-16T16:38:42,665 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-16T16:38:42,665 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-16T16:38:42,666 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-16T16:38:42,667 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-16T16:38:42,667 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,667 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,667 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-16T16:38:42,668 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-16T16:38:42,668 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,668 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,668 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-16T16:38:42,669 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-16T16:38:42,669 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,669 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,669 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-16T16:38:42,670 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-16T16:38:42,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-16T16:38:42,670 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-16T16:38:42,670 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-16T16:38:42,671 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740
2024-11-16T16:38:42,672 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740
2024-11-16T16:38:42,673 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-16T16:38:42,673 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-16T16:38:42,673 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-16T16:38:42,674 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-16T16:38:42,675 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=835535, jitterRate=0.06243769824504852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-16T16:38:42,675 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-16T16:38:42,675 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731775122665Writing region info on filesystem at 1731775122665Initializing all the Stores at 1731775122666 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122666Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122666Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731775122666Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731775122666Cleaning up temporary data from old regions at 1731775122673 (+7 ms)Running coprocessor post-open hooks at 1731775122675 (+2 ms)Region opened successfully at 1731775122675
2024-11-16T16:38:42,676 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731775122644
2024-11-16T16:38:42,678 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-16T16:38:42,678 INFO  [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-16T16:38:42,678 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,679 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,45279,1731775121662, state=OPEN
2024-11-16T16:38:42,734 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:36911/user/jenkins/test-data/1e185121-3c12-1c46-477a-67cdb1b747f4/WALs/303fe01965e8,36407,1731774934631/303fe01965e8%2C36407%2C1731774934631.1731774935277
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor117.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-16T16:38:42,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:38:42,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-16T16:38:42,744 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:42,744 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-16T16:38:42,744 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,748 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-16T16:38:42,748 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,45279,1731775121662 in 256 msec
2024-11-16T16:38:42,752 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-16T16:38:42,752 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 730 msec
2024-11-16T16:38:42,754 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-16T16:38:42,754 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-16T16:38:42,755 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:38:42,755 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,45279,1731775121662, seqNum=-1]
2024-11-16T16:38:42,756 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:38:42,757 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60259, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:38:42,764 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 784 msec
2024-11-16T16:38:42,764 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731775122764, completionTime=-1
2024-11-16T16:38:42,764 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-16T16:38:42,764 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-16T16:38:42,767 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-16T16:38:42,767 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731775182767
2024-11-16T16:38:42,767 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731775242767
2024-11-16T16:38:42,767 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:39085, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,768 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,770 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.044sec
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-16T16:38:42,773 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-16T16:38:42,775 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-16T16:38:42,775 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-16T16:38:42,775 INFO  [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,39085,1731775121470-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-16T16:38:42,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5963ddb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:42,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,39085,-1 for getting cluster id
2024-11-16T16:38:42,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-16T16:38:42,790 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bdd5b43e-59c5-4f24-ae8a-d95771b689f4'
2024-11-16T16:38:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-16T16:38:42,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bdd5b43e-59c5-4f24-ae8a-d95771b689f4"

2024-11-16T16:38:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57758554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,39085,-1]
2024-11-16T16:38:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-16T16:38:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:42,792 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-16T16:38:42,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b9c3297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-16T16:38:42,793 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-16T16:38:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,45279,1731775121662, seqNum=-1]
2024-11-16T16:38:42,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-16T16:38:42,795 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-16T16:38:42,797 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,39085,1731775121470
2024-11-16T16:38:42,797 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-16T16:38:42,799 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-16T16:38:42,799 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-16T16:38:42,801 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs, maxLogs=32
2024-11-16T16:38:42,801 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731775122801
2024-11-16T16:38:42,806 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1/test.com%2C8080%2C1.1731775122801
2024-11-16T16:38:42,807 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45805:45805),(127.0.0.1/127.0.0.1:35441:35441)]
2024-11-16T16:38:42,808 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731775122808
2024-11-16T16:38:42,812 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,812 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,812 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,812 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,812 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,812 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1/test.com%2C8080%2C1.1731775122801 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1/test.com%2C8080%2C1.1731775122808
2024-11-16T16:38:42,813 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45805:45805),(127.0.0.1/127.0.0.1:35441:35441)]
2024-11-16T16:38:42,813 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1/test.com%2C8080%2C1.1731775122801 is not closed yet, will try archiving it next time
2024-11-16T16:38:42,813 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,813 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,813 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,813 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,813 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:42,813 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741835_1011 (size=93)
2024-11-16T16:38:42,814 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741835_1011 (size=93)
2024-11-16T16:38:42,814 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/WALs/test.com,8080,1/test.com%2C8080%2C1.1731775122801 to hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs/test.com%2C8080%2C1.1731775122801
2024-11-16T16:38:42,815 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741836_1012 (size=93)
2024-11-16T16:38:42,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741836_1012 (size=93)
2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs
2024-11-16T16:38:42,818 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731775122808)
2024-11-16T16:38:42,818 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-16T16:38:42,818 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:42,818 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-16T16:38:42,818 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=875791612, stopped=false
2024-11-16T16:38:42,818 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,39085,1731775121470
2024-11-16T16:38:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-16T16:38:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:42,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:42,833 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:38:42,833 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-16T16:38:42,833 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:42,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:42,834 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,45279,1731775121662' *****
2024-11-16T16:38:42,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:42,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-16T16:38:42,834 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-16T16:38:42,834 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,45279,1731775121662
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:38:42,834 INFO  [RS:0;303fe01965e8:45279 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:45279.
2024-11-16T16:38:42,834 DEBUG [RS:0;303fe01965e8:45279 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-16T16:38:42,835 DEBUG [RS:0;303fe01965e8:45279 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:42,835 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-16T16:38:42,835 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-16T16:38:42,835 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-16T16:38:42,835 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-16T16:38:42,835 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-16T16:38:42,835 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-16T16:38:42,835 DEBUG [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-16T16:38:42,835 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-16T16:38:42,835 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-16T16:38:42,835 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-16T16:38:42,835 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-16T16:38:42,835 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-16T16:38:42,835 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-16T16:38:42,853 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/.tmp/ns/c64fd8db69ec4bf38cedc166d8a6f846 is 43, key is default/ns:d/1731775122758/Put/seqid=0
2024-11-16T16:38:42,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741837_1013 (size=5153)
2024-11-16T16:38:42,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741837_1013 (size=5153)
2024-11-16T16:38:42,857 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/.tmp/ns/c64fd8db69ec4bf38cedc166d8a6f846
2024-11-16T16:38:42,861 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/.tmp/ns/c64fd8db69ec4bf38cedc166d8a6f846 as hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/ns/c64fd8db69ec4bf38cedc166d8a6f846
2024-11-16T16:38:42,865 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/ns/c64fd8db69ec4bf38cedc166d8a6f846, entries=2, sequenceid=6, filesize=5.0 K
2024-11-16T16:38:42,866 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false
2024-11-16T16:38:42,866 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-16T16:38:42,870 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-16T16:38:42,870 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-16T16:38:42,870 INFO  [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:42,871 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731775122835Running coprocessor pre-close hooks at 1731775122835Disabling compacts and flushes for region at 1731775122835Disabling writes for close at 1731775122835Obtaining lock to block concurrent updates at 1731775122836 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731775122836Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731775122836Flushing stores of hbase:meta,,1.1588230740 at 1731775122837 (+1 ms)Flushing 1588230740/ns: creating writer at 1731775122837Flushing 1588230740/ns: appending metadata at 1731775122852 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731775122852Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f7a3bc4: reopening flushed file at 1731775122861 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=6, compaction requested=false at 1731775122866 (+5 ms)Writing region close event to WAL at 1731775122867 (+1 ms)Running coprocessor post-close hooks at 1731775122870 (+3 ms)Closed at 1731775122870
2024-11-16T16:38:42,871 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-16T16:38:43,035 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,45279,1731775121662; all regions closed.
2024-11-16T16:38:43,036 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,037 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,037 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,037 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,037 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,042 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741834_1010 (size=1152)
2024-11-16T16:38:43,043 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741834_1010 (size=1152)
2024-11-16T16:38:43,047 DEBUG [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs
2024-11-16T16:38:43,047 INFO  [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C45279%2C1731775121662.meta:.meta(num 1731775122655)
2024-11-16T16:38:43,047 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,047 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,047 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,047 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,048 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741833_1009 (size=93)
2024-11-16T16:38:43,050 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741833_1009 (size=93)
2024-11-16T16:38:43,051 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-16T16:38:43,051 INFO  [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-16T16:38:43,053 DEBUG [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/oldWALs
2024-11-16T16:38:43,053 INFO  [RS:0;303fe01965e8:45279 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog 303fe01965e8%2C45279%2C1731775121662:(num 1731775122181)
2024-11-16T16:38:43,053 DEBUG [RS:0;303fe01965e8:45279 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-16T16:38:43,053 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.LeaseManager(133): Closed leases
2024-11-16T16:38:43,053 INFO  [RS:0;303fe01965e8:45279 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:38:43,054 INFO  [RS:0;303fe01965e8:45279 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-16T16:38:43,054 INFO  [RS:0;303fe01965e8:45279 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:38:43,054 INFO  [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:38:43,054 INFO  [RS:0;303fe01965e8:45279 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45279
2024-11-16T16:38:43,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,45279,1731775121662
2024-11-16T16:38:43,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-16T16:38:43,064 INFO  [RS:0;303fe01965e8:45279 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:38:43,075 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,45279,1731775121662]
2024-11-16T16:38:43,085 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,45279,1731775121662 already deleted, retry=false
2024-11-16T16:38:43,086 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,45279,1731775121662 expired; onlineServers=0
2024-11-16T16:38:43,086 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,39085,1731775121470' *****
2024-11-16T16:38:43,086 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-16T16:38:43,086 INFO  [M:0;303fe01965e8:39085 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-16T16:38:43,086 INFO  [M:0;303fe01965e8:39085 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-16T16:38:43,086 DEBUG [M:0;303fe01965e8:39085 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-16T16:38:43,086 DEBUG [M:0;303fe01965e8:39085 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-16T16:38:43,086 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-16T16:38:43,086 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775121985 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731775121985,5,FailOnTimeoutGroup]
2024-11-16T16:38:43,086 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775121985 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731775121985,5,FailOnTimeoutGroup]
2024-11-16T16:38:43,087 INFO  [M:0;303fe01965e8:39085 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-16T16:38:43,087 INFO  [M:0;303fe01965e8:39085 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-16T16:38:43,087 DEBUG [M:0;303fe01965e8:39085 {}] master.HMaster(1795): Stopping service threads
2024-11-16T16:38:43,087 INFO  [M:0;303fe01965e8:39085 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-16T16:38:43,087 INFO  [M:0;303fe01965e8:39085 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-16T16:38:43,087 INFO  [M:0;303fe01965e8:39085 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-16T16:38:43,087 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-16T16:38:43,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-16T16:38:43,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-16T16:38:43,096 DEBUG [M:0;303fe01965e8:39085 {}] zookeeper.ZKUtil(347): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-16T16:38:43,096 WARN  [M:0;303fe01965e8:39085 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-16T16:38:43,097 INFO  [M:0;303fe01965e8:39085 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/.lastflushedseqids
2024-11-16T16:38:43,107 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741838_1014 (size=99)
2024-11-16T16:38:43,108 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741838_1014 (size=99)
2024-11-16T16:38:43,108 INFO  [M:0;303fe01965e8:39085 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-16T16:38:43,108 INFO  [M:0;303fe01965e8:39085 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-16T16:38:43,109 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-16T16:38:43,109 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:43,109 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:43,109 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-16T16:38:43,109 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:43,109 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-16T16:38:43,130 DEBUG [M:0;303fe01965e8:39085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db53d62a4b9f42ed8b81edc113d49361 is 82, key is hbase:meta,,1/info:regioninfo/1731775122678/Put/seqid=0
2024-11-16T16:38:43,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741839_1015 (size=5672)
2024-11-16T16:38:43,134 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741839_1015 (size=5672)
2024-11-16T16:38:43,135 INFO  [M:0;303fe01965e8:39085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db53d62a4b9f42ed8b81edc113d49361
2024-11-16T16:38:43,154 DEBUG [M:0;303fe01965e8:39085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cac97565554fbf99a7c4eddd326987 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731775122763/Put/seqid=0
2024-11-16T16:38:43,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741840_1016 (size=5275)
2024-11-16T16:38:43,158 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741840_1016 (size=5275)
2024-11-16T16:38:43,158 INFO  [M:0;303fe01965e8:39085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cac97565554fbf99a7c4eddd326987
2024-11-16T16:38:43,175 DEBUG [M:0;303fe01965e8:39085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7028379145f148b7a484c47ea98660c2 is 69, key is 303fe01965e8,45279,1731775121662/rs:state/1731775122032/Put/seqid=0
2024-11-16T16:38:43,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:43,175 INFO  [RS:0;303fe01965e8:45279 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:38:43,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45279-0x101448343d20001, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:43,175 INFO  [RS:0;303fe01965e8:45279 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,45279,1731775121662; zookeeper connection closed.
2024-11-16T16:38:43,176 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c1579c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c1579c
2024-11-16T16:38:43,176 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-16T16:38:43,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741841_1017 (size=5156)
2024-11-16T16:38:43,179 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741841_1017 (size=5156)
2024-11-16T16:38:43,179 INFO  [M:0;303fe01965e8:39085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7028379145f148b7a484c47ea98660c2
2024-11-16T16:38:43,198 DEBUG [M:0;303fe01965e8:39085 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7e1361dcb91b4f14a0e85fa1b78a175c is 52, key is load_balancer_on/state:d/1731775122798/Put/seqid=0
2024-11-16T16:38:43,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741842_1018 (size=5056)
2024-11-16T16:38:43,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741842_1018 (size=5056)
2024-11-16T16:38:43,202 INFO  [M:0;303fe01965e8:39085 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7e1361dcb91b4f14a0e85fa1b78a175c
2024-11-16T16:38:43,207 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db53d62a4b9f42ed8b81edc113d49361 as hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db53d62a4b9f42ed8b81edc113d49361
2024-11-16T16:38:43,211 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db53d62a4b9f42ed8b81edc113d49361, entries=8, sequenceid=29, filesize=5.5 K
2024-11-16T16:38:43,212 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/28cac97565554fbf99a7c4eddd326987 as hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28cac97565554fbf99a7c4eddd326987
2024-11-16T16:38:43,215 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/28cac97565554fbf99a7c4eddd326987, entries=3, sequenceid=29, filesize=5.2 K
2024-11-16T16:38:43,216 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7028379145f148b7a484c47ea98660c2 as hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7028379145f148b7a484c47ea98660c2
2024-11-16T16:38:43,220 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7028379145f148b7a484c47ea98660c2, entries=1, sequenceid=29, filesize=5.0 K
2024-11-16T16:38:43,221 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7e1361dcb91b4f14a0e85fa1b78a175c as hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7e1361dcb91b4f14a0e85fa1b78a175c
2024-11-16T16:38:43,224 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44701/user/jenkins/test-data/ebb99f3b-0720-ff53-2ce6-8a33daf4e15c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7e1361dcb91b4f14a0e85fa1b78a175c, entries=1, sequenceid=29, filesize=4.9 K
2024-11-16T16:38:43,225 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=29, compaction requested=false
2024-11-16T16:38:43,226 INFO  [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-16T16:38:43,226 DEBUG [M:0;303fe01965e8:39085 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731775123109Disabling compacts and flushes for region at 1731775123109Disabling writes for close at 1731775123109Obtaining lock to block concurrent updates at 1731775123109Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731775123109Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731775123110 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731775123111 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731775123111Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731775123130 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731775123130Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731775123139 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731775123154 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731775123154Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731775123161 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731775123174 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731775123174Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731775123183 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731775123197 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731775123197Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ec6d714: reopening flushed file at 1731775123206 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@111309be: reopening flushed file at 1731775123211 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cdaefc2: reopening flushed file at 1731775123215 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e1a9371: reopening flushed file at 1731775123220 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=29, compaction requested=false at 1731775123225 (+5 ms)Writing region close event to WAL at 1731775123226 (+1 ms)Closed at 1731775123226
2024-11-16T16:38:43,227 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,228 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,228 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,228 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,228 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-16T16:38:43,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37663 is added to blk_1073741830_1006 (size=10311)
2024-11-16T16:38:43,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44397 is added to blk_1073741830_1006 (size=10311)
2024-11-16T16:38:43,230 INFO  [M:0;303fe01965e8:39085 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-16T16:38:43,230 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-16T16:38:43,230 INFO  [M:0;303fe01965e8:39085 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39085
2024-11-16T16:38:43,230 INFO  [M:0;303fe01965e8:39085 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-16T16:38:43,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,338 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,339 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,341 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,341 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,342 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,342 INFO  [M:0;303fe01965e8:39085 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-16T16:38:43,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:43,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39085-0x101448343d20000, quorum=127.0.0.1:63592, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-16T16:38:43,368 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,369 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,372 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,374 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-16T16:38:43,379 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e74a3b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:43,380 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b1af14f{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:43,380 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:43,380 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33455caf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:43,380 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d05b3cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:43,381 WARN  [BP-967819845-172.17.0.2-1731775118939 heartbeating to localhost/127.0.0.1:44701 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:38:43,381 WARN  [BP-967819845-172.17.0.2-1731775118939 heartbeating to localhost/127.0.0.1:44701 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-967819845-172.17.0.2-1731775118939 (Datanode Uuid 5b49ba16-c286-4b60-8b34-043c0192a351) service to localhost/127.0.0.1:44701
2024-11-16T16:38:43,381 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:38:43,381 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:38:43,381 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data3/current/BP-967819845-172.17.0.2-1731775118939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:43,382 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data4/current/BP-967819845-172.17.0.2-1731775118939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:43,382 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:38:43,383 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a3cdc90{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-16T16:38:43,383 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2a377aed{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:43,383 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:43,383 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a9c0f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:43,383 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6adc48be{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:43,384 WARN  [BP-967819845-172.17.0.2-1731775118939 heartbeating to localhost/127.0.0.1:44701 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-16T16:38:43,384 WARN  [BP-967819845-172.17.0.2-1731775118939 heartbeating to localhost/127.0.0.1:44701 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-967819845-172.17.0.2-1731775118939 (Datanode Uuid 798945b2-1c8a-47e8-84ae-8eb51c64a38a) service to localhost/127.0.0.1:44701
2024-11-16T16:38:43,384 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-16T16:38:43,385 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-16T16:38:43,385 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data1/current/BP-967819845-172.17.0.2-1731775118939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:43,385 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/cluster_23dd9162-ac5a-8437-5593-cdfd7f50afe3/data/data2/current/BP-967819845-172.17.0.2-1731775118939 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-16T16:38:43,385 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-16T16:38:43,390 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5c4027cc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-16T16:38:43,391 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@51c3fa60{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-16T16:38:43,391 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-16T16:38:43,391 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6abd1419{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-16T16:38:43,391 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7546f00d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ff42849e-5bc1-1550-4244-a23e1e87b41c/hadoop.log.dir/,STOPPED}
2024-11-16T16:38:43,396 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-16T16:38:43,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-16T16:38:43,418 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=270 (was 231)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44701
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44701 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44701
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:44701
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44701
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44701 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44701
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (874278612) connection to localhost/127.0.0.1:44701 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=532 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=193 (was 201), ProcessCount=11 (was 11), AvailableMemoryMB=4270 (was 4275)